#!/usr/bin/env bash
#
# Homelab monitoring, Part 4: Alerts to your phone
# https://www.zadran.net/guides/homelab-monitoring-4-alerts-to-your-phone
# Generated from the guide, last updated 2026-10-13.
#
# NO WARRANTY. This script is provided as is. You are responsible for
# understanding what it does before running it, and for the consequences.
# Take a backup first. See https://www.zadran.net/disclaimer
#
# Destructive steps are left commented out on purpose -- uncomment them
# only once you have understood what they do.
#
# Where the guide writes a file with <<'EOF' and your values are in it,
# this script uses <<EOF so that they expand. Every other $, ` and \
# gets a backslash, so the rest comes out exactly as the guide shows it.

set -euo pipefail

# ---- Set these for your environment ----
# Secret: anyone who knows it reads your alerts
NTFY_TOPIC='myhome-alerts-3f9a1c2b7e4d'
# From Part 1; several as pve|pve2
PVE_NAME='pve'

echo "myhome-alerts-$(openssl rand -hex 6)"

curl -H "Title: Monitoring test" -d "Hello from Grafana's container" ntfy.sh/${NTFY_TOPIC}

cat > /opt/monitoring/grafana/provisioning/alerting/alerting.yaml <<EOF
apiVersion: 1

# ---------- Message format for ntfy ----------
templates:
  - orgId: 1
    name: ntfy
    template: |
      {{ define "ntfy.title" }}{{ if eq .Status "firing" }}🔴 {{ .CommonLabels.alertname }}{{ else }}✅ Resolved: {{ .CommonLabels.alertname }}{{ end }}{{ end }}
      {{ define "ntfy.message" }}{{ range .Alerts }}{{ if eq .Status "firing" }}• {{ else }}✓ {{ end }}{{ .Annotations.summary }}
      {{ end }}{{ end }}

# ---------- Where alerts go ----------
contactPoints:
  - orgId: 1
    name: ntfy-phone
    receivers:
      - uid: ntfy-phone
        type: webhook
        settings:
          url: "https://ntfy.sh/${NTFY_TOPIC}?tpl=yes&t=%7B%7B.title%7D%7D&m=%7B%7B.message%7D%7D"
          httpMethod: POST
          title: '{{ template "ntfy.title" . }}'
          message: '{{ template "ntfy.message" . }}'
        disableResolveMessage: false

# ---------- Send everything to the phone ----------
policies:
  - orgId: 1
    receiver: ntfy-phone
    group_by: ['alertname', 'grafana_folder']
    group_wait: 30s
    group_interval: 5m
    repeat_interval: 4h

# ---------- The alert rules ----------
groups:
  - orgId: 1
    name: homelab
    folder: Homelab Alerts
    interval: 1m
    rules:

      - uid: hl-target-down
        title: Target down
        condition: C
        for: 3m
        noDataState: OK
        execErrState: Error
        labels: {severity: critical}
        annotations:
          summary: '{{ \$labels.job }} target {{ \$labels.instance }} is unreachable'
        data:
          - refId: A
            relativeTimeRange: {from: 300, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: 'up'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: lt, params: [1]}}]

      - uid: hl-pool-full
        title: ZFS pool above 80%
        condition: C
        for: 10m
        noDataState: OK
        execErrState: Error
        labels: {severity: warning}
        annotations:
          summary: 'Pool {{ \$labels.pool }} on {{ \$labels.host }} is {{ printf "%.0f" \$values.A.Value }}% full'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: 'zpool_capacity_percent'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: gt, params: [80]}}]

      - uid: hl-pool-unhealthy
        title: ZFS pool not healthy
        condition: C
        for: 2m
        noDataState: OK
        execErrState: Error
        labels: {severity: critical}
        annotations:
          summary: 'Pool {{ \$labels.pool }} on {{ \$labels.host }} is {{ \$labels.state }}'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: 'zpool_healthy'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: lt, params: [1]}}]

      - uid: hl-cpu-high
        title: High CPU
        condition: C
        for: 15m
        noDataState: OK
        execErrState: Error
        labels: {severity: warning}
        annotations:
          summary: 'CPU on {{ \$labels.host }} at {{ printf "%.0f" \$values.A.Value }}% for 15 min'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: '100 - avg by (host) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: gt, params: [90]}}]

      - uid: hl-ram-high
        title: High RAM on Proxmox host
        condition: C
        for: 15m
        noDataState: OK
        execErrState: Error
        labels: {severity: warning}
        annotations:
          summary: 'RAM on {{ \$labels.host }} at {{ printf "%.0f" \$values.A.Value }}% for 15 min'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: '100 * (1 - node_memory_MemAvailable_bytes{host=~"${PVE_NAME}"} / node_memory_MemTotal_bytes{host=~"${PVE_NAME}"})'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: gt, params: [95]}}]

      - uid: hl-rootfs-full
        title: System disk above 85%
        condition: C
        for: 10m
        noDataState: OK
        execErrState: Error
        labels: {severity: warning}
        annotations:
          summary: 'System disk (/) on {{ \$labels.host }} is {{ printf "%.0f" \$values.A.Value }}% full'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: '100 * (1 - node_filesystem_avail_bytes{mountpoint="/", fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/", fstype!~"tmpfs|overlay"})'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: gt, params: [85]}}]

      - uid: hl-temp-high
        title: Hardware too hot
        condition: C
        for: 10m
        noDataState: OK
        execErrState: Error
        labels: {severity: warning}
        annotations:
          summary: '{{ \$labels.host }} {{ \$labels.chip }} {{ \$labels.sensor }} at {{ printf "%.0f" \$values.A.Value }} °C'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: 'node_hwmon_temp_celsius{chip=~".*coretemp.*|nvme.*"}'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: gt, params: [85]}}]

      - uid: hl-unifi-offline
        title: UniFi device offline
        condition: C
        for: 3m
        noDataState: OK
        execErrState: Error
        labels: {severity: critical}
        annotations:
          summary: 'UniFi {{ \$labels.type }} "{{ \$labels.name }}" stopped reporting'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: 'changes(unpoller_device_uptime_seconds[5m])'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: lt, params: [1]}}]
EOF
cd /opt/monitoring && docker compose restart grafana
sleep 20
docker compose logs grafana --since 1m 2>&1 | grep -iE 'level=error.*(alert|provision)' || echo "no errors"

# !! DESTRUCTIVE -- review, then uncomment to run:
# systemctl stop prometheus-node-exporter

systemctl start prometheus-node-exporter