#!/usr/bin/env bash
# =============================================================================
#  Zadran Lab – Monitoring stack setup (Prometheus + Grafana + exporters)
#
#  Builds the complete stack from scratch on a Debian 12/13 container or VM:
#    - Docker (installed if missing), jq, curl
#    - compose.yaml, prometheus.yml, exporter configs, secret files
#    - Grafana data source, alert rules (ntfy), dashboards in folders
#
#  Usage:   1. put lab.env next to this script and fill it in
#           2. bash setup-monitoring.sh
#  Re-running is safe: configs are rewritten, Prometheus/Grafana data
#  (Docker volumes) are kept, the previous config folder is backed up.
#
#  Option:  SKIP_DOCKER=1 bash setup-monitoring.sh   -> only generate files
# =============================================================================
set -euo pipefail

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ENV_FILE="${ENV_FILE:-$HERE/lab.env}"
UNPOLLER_DASH_COMMIT="adbe342f09d0fe06b1cc152d28ab0bae5ee3d8b9"   # github.com/unpoller/dashboards, folder v2.0.0
PROXMOX_DASH_REV="4"                                               # grafana.com dashboard 10347

c_ok()   { printf '\e[32m✔ %s\e[0m\n' "$*"; }
c_info() { printf '\e[36m➜ %s\e[0m\n' "$*"; }
c_warn() { printf '\e[33m⚠ %s\e[0m\n' "$*"; }
die()    { printf '\e[31m✘ %s\e[0m\n' "$*" >&2; exit 1; }

# ---------------------------------------------------------------- 0. settings
[[ -f "$ENV_FILE" ]] || die "Settings file not found: $ENV_FILE"
# shellcheck disable=SC1090
source "$ENV_FILE"

: "${STACK_DIR:=/opt/monitoring}" "${TZ:=Europe/Berlin}" "${RETENTION:=730d}"
: "${NODE_TARGETS:=}" "${SLEEPING_HOSTS:=}" "${PVE_NODES:=}" "${PVE_USER:=prometheus@pve}" "${PVE_TOKEN_NAME:=exporter}"
: "${UNIFI_URL:=}" "${UNIFI_USER:=unpoller}" "${UNIFI_PASS:=}"
: "${HA_TARGET:=}" "${HA_TOKEN:=}" "${HA_DROP_REGEX:=}" "${HA_ALERTS:=no}"
: "${WINDOWS_TARGETS:=}" "${KUMA_TARGET:=}" "${KUMA_KEY:=}"
: "${DASH_SOURCE:=}" "${NTFY_URL:=}" "${GRAFANA_ADMIN_PASSWORD:=}"

[[ "${SKIP_DOCKER:-0}" == 1 || $EUID -eq 0 ]] || die "Run as root."
[[ "$STACK_DIR" == /*/* ]]         || die "STACK_DIR must be an absolute path like /opt/monitoring"
[[ -n "$GRAFANA_ADMIN_PASSWORD" ]] || die "GRAFANA_ADMIN_PASSWORD is empty in lab.env"
[[ -n "$NTFY_URL" ]]               || die "NTFY_URL is empty in lab.env"
[[ -n "$NODE_TARGETS" ]]           || die "NODE_TARGETS is empty in lab.env"
[[ -z "$UNIFI_URL"   || -n "$UNIFI_PASS" ]] || die "UNIFI_URL is set but UNIFI_PASS is empty"
[[ -z "$HA_TARGET"   || -n "$HA_TOKEN"   ]] || die "HA_TARGET is set but HA_TOKEN is empty"
[[ -z "$KUMA_TARGET" || -n "$KUMA_KEY"   ]] || die "KUMA_TARGET is set but KUMA_KEY is empty"
for n in $PVE_NODES; do [[ "$n" =~ ^[A-Za-z0-9_-]+=[0-9.]+=.+$ ]] || die "PVE_NODES entry needs name=ip=token: $n"; done

# ---------------------------------------------------------------- 1. tools
if [[ "${SKIP_DOCKER:-0}" != 1 ]]; then
  c_info "Installing tools (curl, jq, Docker if missing)"
  apt-get update -qq && apt-get install -y -qq curl jq ca-certificates >/dev/null
  command -v docker >/dev/null || curl -fsSL https://get.docker.com | sh >/dev/null
  c_ok "Docker $(docker version --format '{{.Server.Version}}')"
else
  command -v jq >/dev/null || die "jq is required"
fi

# ---------------------------------------------------------------- 2. folders
if [[ -d "$STACK_DIR" ]]; then
  BK="$STACK_DIR.bak-$(date +%Y%m%d-%H%M%S)"
  cp -a "$STACK_DIR" "$BK" && c_ok "Previous config backed up to $BK"
fi
P="$STACK_DIR/prometheus"; G="$STACK_DIR/grafana/provisioning"; DB="$G/dashboards/files"
mkdir -p "$P" "$G/datasources" "$G/alerting" "$G/plugins" "$DB/Overview" "$DB/Servers" "$DB/Network" "$DB/Home"
find "$G/alerting" -maxdepth 1 -name '*.yaml' -delete      # old rule files are recreated below
find "$DB" -mindepth 2 -maxdepth 2 -name '*.json' -delete   # old dashboard files are recreated below

# ---------------------------------------------------------------- 3. secrets
umask 077
cat > "$STACK_DIR/.env" <<EOF
GRAFANA_ADMIN_PASSWORD=$GRAFANA_ADMIN_PASSWORD
UNPOLLER_PASSWORD=$UNIFI_PASS
EOF
: > "$P/pve.yml"
for n in $PVE_NODES; do
  IFS='=' read -r mod ip tok <<<"$n"
  printf '%s:\n  user: %s\n  token_name: %s\n  token_value: %s\n  verify_ssl: false\n' \
    "$mod" "$PVE_USER" "$PVE_TOKEN_NAME" "$tok" >> "$P/pve.yml"
done
[[ -n "$HA_TOKEN" ]] && printf '%s' "$HA_TOKEN" > "$P/ha_token"
[[ -n "$KUMA_KEY" ]] && printf '%s' "$KUMA_KEY" > "$P/kuma_key"
umask 022
chmod 644 "$P/pve.yml"
[[ -f "$P/ha_token" ]] && chmod 644 "$P/ha_token"
[[ -f "$P/kuma_key" ]] && chmod 644 "$P/kuma_key"
c_ok "Secret files written (.env is root-only)"

# ---------------------------------------------------------------- 4. compose.yaml
{
cat <<EOF
services:
  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    restart: unless-stopped
    command:
      - --config.file=/etc/prometheus/prometheus.yml
      - --storage.tsdb.path=/prometheus
      - --storage.tsdb.retention.time=$RETENTION
      - --web.enable-lifecycle
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus:/etc/prometheus:ro
      - prometheus-data:/prometheus

  grafana:
    image: grafana/grafana-oss:latest
    container_name: grafana
    restart: unless-stopped
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=\${GRAFANA_ADMIN_PASSWORD}
      - GF_USERS_ALLOW_SIGN_UP=false
      - GF_PLUGINS_PREINSTALL_DISABLED=true
      - TZ=$TZ
    volumes:
      - grafana-data:/var/lib/grafana
      - ./grafana/provisioning:/etc/grafana/provisioning:ro
    depends_on:
      - prometheus
EOF
if [[ -n "$PVE_NODES" ]]; then cat <<EOF

  pve-exporter:
    image: prompve/prometheus-pve-exporter:latest
    container_name: pve-exporter
    restart: unless-stopped
    volumes:
      - ./prometheus/pve.yml:/etc/prometheus/pve.yml:ro
EOF
fi
if [[ -n "$UNIFI_URL" ]]; then cat <<EOF

  unpoller:
    image: ghcr.io/unpoller/unpoller:latest
    container_name: unpoller
    restart: unless-stopped
    environment:
      - UP_UNIFI_DEFAULT_URL=$UNIFI_URL
      - UP_UNIFI_DEFAULT_USER=$UNIFI_USER
      - UP_UNIFI_DEFAULT_PASS=\${UNPOLLER_PASSWORD}
      - UP_UNIFI_DEFAULT_VERIFY_SSL=false
      - UP_UNIFI_DEFAULT_SAVE_SITES=true
      - UP_UNIFI_DEFAULT_SAVE_DPI=false
      - UP_INFLUXDB_DISABLE=true
      - UP_PROMETHEUS_DISABLE=false
      - UP_PROMETHEUS_NAMESPACE=unpoller
      - TZ=$TZ
EOF
fi
cat <<EOF

volumes:
  prometheus-data:
  grafana-data:
EOF
} > "$STACK_DIR/compose.yaml"
c_ok "compose.yaml"

# ---------------------------------------------------------------- 5. prometheus.yml
targets_block() {   # $1 = "name=address ..."  -> static_configs entries with a host label
  for t in $1; do printf "      - targets: ['%s']\n        labels:\n          host: %s\n" "${t#*=}" "${t%%=*}"; done
}
{
cat <<'EOF'
global:
  scrape_interval: 30s
  evaluation_interval: 30s

scrape_configs:
  - job_name: prometheus
    static_configs:
      - targets: ['localhost:9090']

  - job_name: node
    static_configs:
EOF
targets_block "$(for t in $NODE_TARGETS; do printf '%s:9100 ' "$t"; done)"
if [[ -n "$PVE_NODES" ]]; then
cat <<'EOF'

  - job_name: pve
    metrics_path: /pve
    params:
      cluster: ['1']
      node: ['1']
    static_configs:
EOF
for n in $PVE_NODES; do IFS='=' read -r mod ip _ <<<"$n"
  printf "      - targets: ['%s']\n        labels:\n          module: %s\n" "$ip" "$mod"; done
cat <<'EOF'
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [module]
        target_label: __param_module
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: pve-exporter:9221
EOF
fi
if [[ -n "$UNIFI_URL" ]]; then
cat <<'EOF'

  - job_name: unpoller
    scrape_interval: 30s
    static_configs:
      - targets: ['unpoller:9130']
EOF
fi
if [[ -n "$HA_TARGET" ]]; then
cat <<EOF

  - job_name: homeassistant
    scrape_interval: 60s
    metrics_path: /api/prometheus
    authorization:
      credentials_file: ha_token
    static_configs:
      - targets: ['$HA_TARGET']
        labels:
          host: homeassistant
EOF
if [[ -n "$HA_DROP_REGEX" ]]; then cat <<EOF
    metric_relabel_configs:
      - source_labels: [entity]
        regex: '$HA_DROP_REGEX'
        action: drop
EOF
fi
fi
if [[ -n "$WINDOWS_TARGETS" ]]; then
printf '\n  - job_name: windows\n    static_configs:\n'
targets_block "$(for t in $WINDOWS_TARGETS; do printf '%s:9182 ' "$t"; done)"
fi
if [[ -n "$KUMA_TARGET" ]]; then
cat <<EOF

  - job_name: uptimekuma
    scrape_interval: 60s
    basic_auth:
      username: prometheus
      password_file: kuma_key
    static_configs:
      - targets: ['$KUMA_TARGET']
        labels:
          host: uptime
EOF
fi
} > "$P/prometheus.yml"
c_ok "prometheus.yml"

# ---------------------------------------------------------------- 6. data source + dashboard provider
cat > "$G/datasources/prometheus.yml" <<'EOF'
apiVersion: 1
datasources:
  - name: Prometheus
    type: prometheus
    uid: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
EOF
cat > "$G/dashboards/dashboards.yaml" <<'EOF'
apiVersion: 1
providers:
  - name: zadran-lab
    orgId: 1
    type: file
    disableDeletion: true
    allowUiUpdates: false
    updateIntervalSeconds: 60
    options:
      path: /etc/grafana/provisioning/dashboards/files
      foldersFromFilesStructure: true
EOF
c_ok "Data source + dashboard provider"

# ---------------------------------------------------------------- 7. alert rules
sed_escape() { printf '%s' "$1" | sed 's/[&|\\/]/\\&/g'; }
SLEEP_RE=$(echo $SLEEPING_HOSTS | tr ' ' '|')
if [[ -n "$SLEEP_RE" ]]; then TD_EXPR="up{host!~\"$SLEEP_RE\"}"; else TD_EXPR="up"; fi
PVE_RE=$(for n in $PVE_NODES; do printf '%s|' "${n%%=*}"; done); PVE_RE=${PVE_RE%|}; PVE_RE=${PVE_RE:-none}

cat > "$G/alerting/alerting.yaml" <<'__EOF__'
apiVersion: 1

# ---------- Message format for ntfy ----------
templates:
  - orgId: 1
    name: ntfy
    template: |
      {{ define "ntfy.title" }}{{ if eq .Status "firing" }}🔴 {{ .CommonLabels.alertname }}{{ else }}✅ Resolved: {{ .CommonLabels.alertname }}{{ end }}{{ end }}
      {{ define "ntfy.message" }}{{ range .Alerts }}{{ if eq .Status "firing" }}• {{ else }}✓ {{ end }}{{ .Annotations.summary }}
      {{ end }}{{ end }}

# ---------- Where alerts go ----------
contactPoints:
  - orgId: 1
    name: ntfy-phone
    receivers:
      - uid: ntfy-phone
        type: webhook
        settings:
          url: "__NTFY_URL__?tpl=yes&t=%7B%7B.title%7D%7D&m=%7B%7B.message%7D%7D"
          httpMethod: POST
          title: '{{ template "ntfy.title" . }}'
          message: '{{ template "ntfy.message" . }}'
        disableResolveMessage: false

# ---------- Send everything to the phone ----------
policies:
  - orgId: 1
    receiver: ntfy-phone
    group_by: ['alertname', 'grafana_folder']
    group_wait: 30s
    group_interval: 5m
    repeat_interval: 4h

# ---------- The alert rules ----------
groups:
  - orgId: 1
    name: homelab
    folder: Homelab Alerts
    interval: 1m
    rules:

      - uid: hl-target-down
        title: Target down
        condition: C
        for: 3m
        noDataState: OK
        execErrState: Error
        labels: {severity: critical}
        annotations:
          summary: '{{ $labels.job }} target {{ $labels.instance }} is unreachable'
        data:
          - refId: A
            relativeTimeRange: {from: 300, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: '__TARGET_DOWN_EXPR__'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: lt, params: [1]}}]

      - uid: hl-pool-full
        title: ZFS pool above 80%
        condition: C
        for: 10m
        noDataState: OK
        execErrState: Error
        labels: {severity: warning}
        annotations:
          summary: 'Pool {{ $labels.pool }} on {{ $labels.host }} is {{ printf "%.0f" $values.A.Value }}% full'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: 'zpool_capacity_percent'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: gt, params: [80]}}]

      - uid: hl-pool-unhealthy
        title: ZFS pool not healthy
        condition: C
        for: 2m
        noDataState: OK
        execErrState: Error
        labels: {severity: critical}
        annotations:
          summary: 'Pool {{ $labels.pool }} on {{ $labels.host }} is {{ $labels.state }}'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: 'zpool_healthy'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: lt, params: [1]}}]

      - uid: hl-cpu-high
        title: High CPU
        condition: C
        for: 15m
        noDataState: OK
        execErrState: Error
        labels: {severity: warning}
        annotations:
          summary: 'CPU on {{ $labels.host }} at {{ printf "%.0f" $values.A.Value }}% for 15 min'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: '100 - avg by (host) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: gt, params: [90]}}]

      - uid: hl-ram-high
        title: High RAM on Proxmox host
        condition: C
        for: 15m
        noDataState: OK
        execErrState: Error
        labels: {severity: warning}
        annotations:
          summary: 'RAM on {{ $labels.host }} at {{ printf "%.0f" $values.A.Value }}% for 15 min'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: '100 * (1 - node_memory_MemAvailable_bytes{host=~"__PVE_HOSTS__"} / node_memory_MemTotal_bytes{host=~"__PVE_HOSTS__"})'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: gt, params: [95]}}]

      - uid: hl-rootfs-full
        title: System disk above 85%
        condition: C
        for: 10m
        noDataState: OK
        execErrState: Error
        labels: {severity: warning}
        annotations:
          summary: 'System disk (/) on {{ $labels.host }} is {{ printf "%.0f" $values.A.Value }}% full'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: '100 * (1 - node_filesystem_avail_bytes{mountpoint="/", fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/", fstype!~"tmpfs|overlay"})'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: gt, params: [85]}}]

      - uid: hl-temp-high
        title: Hardware too hot
        condition: C
        for: 10m
        noDataState: OK
        execErrState: Error
        labels: {severity: warning}
        annotations:
          summary: '{{ $labels.host }} {{ $labels.chip }} {{ $labels.sensor }} at {{ printf "%.0f" $values.A.Value }} °C'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: 'node_hwmon_temp_celsius{chip=~".*coretemp.*|nvme.*"}'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: gt, params: [85]}}]

      - uid: hl-unifi-offline
        title: UniFi device offline
        condition: C
        for: 3m
        noDataState: OK
        execErrState: Error
        labels: {severity: critical}
        annotations:
          summary: 'UniFi {{ $labels.type }} "{{ $labels.name }}" stopped reporting'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: 'changes(unpoller_device_uptime_seconds[5m])'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: lt, params: [1]}}]
__EOF__
sed -i -e "s|__NTFY_URL__|$(sed_escape "$NTFY_URL")|" \
       -e "s|__TARGET_DOWN_EXPR__|$(sed_escape "$TD_EXPR")|" \
       -e "s|__PVE_HOSTS__|$(sed_escape "$PVE_RE")|g" "$G/alerting/alerting.yaml"

if [[ -n "$WINDOWS_TARGETS" ]]; then
cat > "$G/alerting/windows.yaml" <<'__EOF__'
apiVersion: 1

# ---------- Windows servers (DC1, CM01) — uses contact point + policy from alerting.yaml ----------
groups:
  - orgId: 1
    name: windows
    folder: Homelab Alerts
    interval: 1m
    rules:

      - uid: win-service-stopped
        title: Windows service stopped
        condition: C
        for: 3m
        noDataState: OK
        execErrState: Error
        labels: {severity: critical}
        annotations:
          summary: '{{ $labels.name }} is stopped on {{ $labels.host }}'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: '(windows_service_state{state="running"} == bool 0) and on(instance, name) (windows_service_start_mode{start_mode="auto"} == 1)'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: gt, params: [0]}}]

      - uid: win-disk-full
        title: Windows disk above 85%
        condition: C
        for: 10m
        noDataState: OK
        execErrState: Error
        labels: {severity: warning}
        annotations:
          summary: 'Drive {{ $labels.volume }} on {{ $labels.host }} is {{ printf "%.0f" $values.A.Value }}% full'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: '100 * (1 - windows_logical_disk_free_bytes{volume=~"[A-Z]:"} / windows_logical_disk_size_bytes{volume=~"[A-Z]:"})'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: gt, params: [85]}}]

      - uid: win-cpu-high
        title: Windows CPU high
        condition: C
        for: 15m
        noDataState: OK
        execErrState: Error
        labels: {severity: warning}
        annotations:
          summary: 'CPU on {{ $labels.host }} at {{ printf "%.0f" $values.A.Value }}% for 15 min'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: '100 - avg by (host) (rate(windows_cpu_time_total{mode="idle"}[5m])) * 100'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: gt, params: [90]}}]

      - uid: win-time-drift
        title: Domain clock out of sync
        condition: C
        for: 10m
        noDataState: OK
        execErrState: Error
        labels: {severity: critical}
        annotations:
          summary: 'Clock on {{ $labels.host }} is {{ printf "%.0f" $values.A.Value }} s off. Kerberos logons fail above 5 min'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: 'abs(windows_time_computed_time_offset_seconds)'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: gt, params: [60]}}]

      - uid: win-ad-replication
        title: AD replication backlog
        condition: C
        for: 30m
        noDataState: OK
        execErrState: Error
        labels: {severity: warning}
        annotations:
          summary: '{{ $labels.host }} has {{ printf "%.0f" $values.A.Value }} pending AD replication syncs for 30 min'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: 'sum by (host) (windows_ad_replication_pending_synchronizations)'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: gt, params: [0]}}]

      - uid: win-sql-log-full
        title: SCCM database log almost full
        condition: C
        for: 15m
        noDataState: OK
        execErrState: Error
        labels: {severity: warning}
        annotations:
          summary: 'SQL log of {{ $labels.database }} on {{ $labels.host }} is {{ printf "%.0f" $values.A.Value }}% used'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: 'max by (host, database) (windows_mssql_databases_log_used_percent{database=~"CM_.*"})'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: gt, params: [90]}}]
__EOF__
fi
if [[ -n "$HA_TARGET" && "$HA_ALERTS" == "yes" ]]; then
cat > "$G/alerting/homeassistant.yaml" <<'__EOF__'
apiVersion: 1

# ---------- Home Assistant alert rules (uses contact point + policy from alerting.yaml) ----------
groups:
  - orgId: 1
    name: home
    folder: Homelab Alerts
    interval: 1m
    rules:

      - uid: ha-rack-hot
        title: Network rack too hot
        condition: C
        for: 10m
        noDataState: OK
        execErrState: Error
        labels: {severity: critical}
        annotations:
          summary: 'Network rack is at {{ printf "%.1f" $values.A.Value }} °C'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: 'hass_sensor_temperature_celsius{entity="sensor.thermo_network_rack_temperature"}'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: gt, params: [35]}}]

      - uid: ha-humidity-high
        title: Humidity high (mould risk)
        condition: C
        for: 2h
        noDataState: OK
        execErrState: Error
        labels: {severity: warning}
        annotations:
          summary: '{{ $labels.friendly_name }} at {{ printf "%.0f" $values.A.Value }} % for 2 hours. Air the room'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: '{__name__=~"hass_sensor_(humidity|unit)_percent", entity=~"sensor.*_humidity", entity!="sensor.thermo_outdoor_humidity"}'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: gt, params: [65]}}]

      - uid: ha-battery-low
        title: Sensor battery low
        condition: C
        for: 1h
        noDataState: OK
        execErrState: Error
        labels: {severity: warning}
        annotations:
          summary: '{{ $labels.friendly_name }} at {{ printf "%.0f" $values.A.Value }} %. Replace the battery soon'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: 'hass_sensor_battery_percent{entity=~"sensor.(thermo_|h5075_).*"}'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: lt, params: [15]}}]

      - uid: ha-filter-low
        title: Air purifier filter almost used up
        condition: C
        for: 1h
        noDataState: OK
        execErrState: Error
        labels: {severity: warning}
        annotations:
          summary: '{{ $labels.entity }} filter at {{ printf "%.0f" $values.A.Value }} %. Order a new filter'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: '{__name__=~"hass_sensor_.+", entity=~"sensor.air_purifier_filter_lifetime.*"}'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: lt, params: [10]}}]

      - uid: ha-zigbee-down
        title: Zigbee network offline
        condition: C
        for: 5m
        noDataState: OK
        execErrState: Error
        labels: {severity: critical}
        annotations:
          summary: 'Zigbee2MQTT bridge lost connection. Zigbee sensors and plugs are not reporting'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: 'hass_binary_sensor_state{entity="binary_sensor.zigbee2mqtt_bridge_connection_state"}'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: lt, params: [1]}}]

      - uid: ha-power-cut
        title: Power cut (UPS on battery)
        condition: C
        for: 1m
        noDataState: OK
        execErrState: Error
        labels: {severity: critical}
        annotations:
          summary: '{{ $labels.friendly_name }} lost its input power. Running on battery'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: '(hass_sensor_voltage_v{entity="sensor.ugreen_input_voltage"} < bool 10) or (hass_sensor_voltage_v{entity="sensor.ugreen_usp_input_voltage"} < bool 180)'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: gt, params: [0]}}]
__EOF__
fi
c_ok "Alert rules: $(ls "$G/alerting" | tr '\n' ' ')"

# ---------------------------------------------------------------- 8. dashboards
TMP=$(mktemp -d)
fetch() { curl -fsSL --retry 2 -o "$2" "$1"; }
fix_ds() {   # point the import placeholder at our data source, drop import metadata
  jq 'walk(if type == "string" then gsub("\\$\\{DS_PROMETHEUS\\}"; "prometheus") else . end)
      | del(.__inputs, .__requires, .__elements) | .id = null' "$1"
}

# 8a. Node Exporter Full (grafana.com 1860) – used unchanged
if fetch "https://grafana.com/api/dashboards/1860/revisions/latest/download" "$TMP/1860.json"; then
  fix_ds "$TMP/1860.json" > "$DB/Servers/node-exporter-full.json" && c_ok "Node Exporter Full (1860)"
else c_warn "Could not download dashboard 1860 from grafana.com"; fi

# 8b. Proxmox via Prometheus (grafana.com 10347) – CPU panels switched to node_exporter
if [[ -n "$PVE_NODES" ]]; then
  if fetch "https://grafana.com/api/dashboards/10347/revisions/$PROXMOX_DASH_REV/download" "$TMP/10347.json"; then
    CPU='100 - avg(rate(node_cpu_seconds_total{mode="idle", instance=~"$instance:.*"}[1m])) * 100'
    fix_ds "$TMP/10347.json" | jq --arg cpu "$CPU" '
        (.. | objects | select(.id? == 7 and has("targets"))) |=
            (.targets |= [.[0] | .expr = $cpu]
             | .fieldConfig.defaults.unit = "percent" | .fieldConfig.defaults.max = 100
             | .fieldConfig.defaults.thresholds.steps[1].value = 70
             | .fieldConfig.defaults.thresholds.steps[2].value = 90)
      | (.. | objects | select(.id? == 23 and has("targets"))) |=
            (.targets |= [.[0] | .expr = $cpu]
             | .fieldConfig.defaults.unit = "percent" | .fieldConfig.defaults.min = 0 | .fieldConfig.defaults.max = 100)
      | (.. | objects | select(.id? == 22 and has("targets"))) |=
            (.targets[0].expr = "count(count by (cpu) (node_cpu_seconds_total{instance=~\"$instance:.*\"}))")
    ' > "$DB/Servers/proxmox.json"
    n=$(grep -c 'node_cpu_seconds_total' "$DB/Servers/proxmox.json" || true)
    if [[ "$n" -ge 3 ]]; then c_ok "Proxmox via Prometheus (10347 rev $PROXMOX_DASH_REV, CPU panels patched)"
    else c_warn "Proxmox dashboard: CPU patch did not apply (panel ids changed?)"; fi
  else c_warn "Could not download dashboard 10347 from grafana.com"; fi
fi

# 8c. UniFi-Poller v2.0.0 (GitHub) – data source fixed, UAP panels corrected
if [[ -n "$UNIFI_URL" ]]; then
  RAW="https://raw.githubusercontent.com/unpoller/dashboards/$UNPOLLER_DASH_COMMIT/v2.0.0"
  for pair in "Client%20Insights:unifi-clients" "Network%20Sites:unifi-sites" "UAP%20Insights:unifi-uap" "USW%20Insights:unifi-usw"; do
    name=${pair%%:*}; out=${pair#*:}
    if fetch "$RAW/UniFi-Poller_%20$name%20-%20Prometheus.json" "$TMP/$out.json"; then
      fix_ds "$TMP/$out.json" > "$DB/Network/$out.json"
    else c_warn "Could not download UniFi dashboard ${name//%20/ }"; fi
  done
  if [[ -f "$DB/Network/unifi-uap.json" ]]; then
    jq '
        walk(if type == "object" and has("panels") then .panels |= map(select(.id != 32)) else . end)
      | (.. | objects | select(.id? == 48 and has("targets"))) |=
            (.targets[0].expr = "count(unpoller_client_uptime_seconds{site_name=~\"$Site\", ap_name=~\"$AP\", wired=\"false\"}) or vector(0)")
      | (.. | objects | select(.id? == 60 and has("targets"))) |=
            (.targets[0].expr = "count(unpoller_client_uptime_seconds{site_name=~\"$Site\", ap_name=~\"$AP\", wired=\"false\", essid=~\"(?i).*guest.*\"}) or vector(0)")
      | (.. | objects | select((.id? == 52 or .id? == 54 or .id? == 56) and has("targets"))) |=
            (.targets |= map(.instant = true | .range = false))
    ' "$DB/Network/unifi-uap.json" > "$TMP/uap.json" && mv "$TMP/uap.json" "$DB/Network/unifi-uap.json"
    c_ok "UniFi-Poller dashboards (UAP: users/guests fixed, pies instant, clock panel removed)"
  fi
fi

# 8d. Zadran Lab dashboards – from a local folder (relative to this script) or a URL
if [[ -n "$DASH_SOURCE" && ! "$DASH_SOURCE" =~ ^https?:// && "$DASH_SOURCE" != /* ]]; then DASH_SOURCE="$HERE/$DASH_SOURCE"; fi
own() {   # $1 = file name, $2 = folder
  [[ -n "$DASH_SOURCE" ]] || return 0
  if [[ "$DASH_SOURCE" =~ ^https?:// ]]; then
    fetch "$DASH_SOURCE/$1" "$DB/$2/$1" || { c_warn "Could not download $1"; return 0; }
  else
    cp "$DASH_SOURCE/$1" "$DB/$2/$1" 2>/dev/null || { c_warn "Missing $DASH_SOURCE/$1"; return 0; }
  fi
  c_ok "$1"
}
own zadran-lab-overview.json Overview
own nas-pools.json           Servers
if [[ -n "$WINDOWS_TARGETS" ]]; then own windows-servers.json Servers; fi
if [[ -n "$HA_TARGET" ]];       then own home-assistant.json  Home;    fi
find "$TMP" -mindepth 1 -delete; rmdir "$TMP"
find "$DB" -mindepth 1 -type d -empty -delete    # no empty folders in Grafana

# ---------------------------------------------------------------- 9. start
if [[ "${SKIP_DOCKER:-0}" == 1 ]]; then c_ok "Files generated in $STACK_DIR (SKIP_DOCKER=1, nothing started)"; exit 0; fi
cd "$STACK_DIR"
docker compose pull -q
docker compose up -d --remove-orphans
docker compose restart grafana >/dev/null      # re-read provisioning on re-runs
curl -s -X POST localhost:9090/-/reload >/dev/null || true
c_info "Waiting for Grafana and the first scrapes (60 s) ..."
sleep 60
echo
echo "================ Targets ================"
curl -s 'localhost:9090/api/v1/targets?state=active' |
  jq -r '.data.activeTargets[] | "\(if .health == "up" then "UP" else "DOWN" end)\t\(.labels.job)\t\(.labels.host // .labels.instance)"' |
  sort -k2 | column -t -s $'\t'
echo "================ Dashboards ============="
curl -s -u "admin:$GRAFANA_ADMIN_PASSWORD" 'localhost:3000/api/search?type=dash-db&limit=500' |
  jq -r '.[] | "\(.folderTitle // "General")\t\(.title)"' | sort | column -t -s $'\t'
echo
c_ok "Done. Grafana: http://$(hostname -I | awk '{print $1}'):3000  (user: admin)"