#!/usr/bin/env bash
#
# Homelab monitoring, Part 5: Home Assistant, Windows servers, Uptime Kuma, and a one-command rebuild
# https://www.zadran.net/guides/homelab-monitoring-5-home-assistant-windows-setup-script
# Generated from the guide, last updated 2026-10-13.
#
# NO WARRANTY. This script is provided as is. You are responsible for
# understanding what it does before running it, and for the consequences.
# Take a backup first. See https://www.zadran.net/disclaimer
#
# Destructive steps are left commented out on purpose -- uncomment them
# only once you have understood what they do.
#
# Where the guide writes a file with <<'EOF' and your values are in it,
# this script uses <<EOF so that they expand. Every other $, ` and \
# gets a backslash, so the rest comes out exactly as the guide shows it.
#
# These are the guide's bash commands. Its PowerShell commands are in
# https://www.zadran.net/guides/homelab-monitoring-5-home-assistant-windows-setup-script/script.ps1

set -euo pipefail

# ---- Set these for your environment ----
# The IP of Home Assistant
HA_IP='192.168.1.20'
# e.g. your domain controller
DC_IP='192.168.1.21'
# How it is labelled on dashboards and alerts
DC_NAME='dc1'
# The IP of Uptime Kuma
KUMA_IP='192.168.1.50'

# >> By hand: the guide has a yaml block here, to type or paste into
# >> a file or a form. It is not a command, so it is not in this script.

ha core check

nano /opt/monitoring/prometheus/ha_token

cd /opt/monitoring/prometheus
tr -cd 'A-Za-z0-9._-' < ha_token > ha_token.tmp && mv ha_token.tmp ha_token && chmod 644 ha_token
curl -s -o /dev/null -w '%{http_code}\n' -H "Authorization: Bearer $(cat ha_token)" http://${HA_IP}:8123/api/prometheus

cat >> /opt/monitoring/prometheus/prometheus.yml <<EOF

  - job_name: homeassistant
    scrape_interval: 60s
    metrics_path: /api/prometheus
    authorization:
      credentials_file: ha_token
    static_configs:
      - targets: ['${HA_IP}:8123']
        labels:
          host: homeassistant
    metric_relabel_configs:
      - source_labels: [entity]
        regex: 'sensor\\.(my_phone|my_tablet)_.*'
        action: drop
EOF
curl -s -X POST localhost:9090/-/reload && echo "reload OK"

cd /opt/monitoring/prometheus
curl -s -H "Authorization: Bearer $(cat ha_token)" http://${HA_IP}:8123/api/prometheus |
  grep -o 'entity="[^"]*",friendly_name="[^"]*"' | sort -u

mkdir -p /opt/monitoring/grafana/provisioning/dashboards/files/Home
curl -fsSL -o /opt/monitoring/grafana/provisioning/dashboards/files/Home/home-assistant.json \
  https://www.zadran.net/downloads/grafana/dashboards/home-assistant.json

cat >> /opt/monitoring/prometheus/prometheus.yml <<EOF

  - job_name: windows
    static_configs:
      - targets: ['${DC_IP}:9182']
        labels:
          host: ${DC_NAME}
EOF
curl -s -X POST localhost:9090/-/reload && echo "reload OK"

curl -fsSL -o /opt/monitoring/grafana/provisioning/dashboards/files/Servers/windows-servers.json \
  https://www.zadran.net/downloads/grafana/dashboards/windows-servers.json

cat > /opt/monitoring/grafana/provisioning/alerting/windows.yaml <<'EOF'
apiVersion: 1

# ---------- Windows servers (DC1, CM01) — uses contact point + policy from alerting.yaml ----------
groups:
  - orgId: 1
    name: windows
    folder: Homelab Alerts
    interval: 1m
    rules:

      - uid: win-service-stopped
        title: Windows service stopped
        condition: C
        for: 3m
        noDataState: OK
        execErrState: Error
        labels: {severity: critical}
        annotations:
          summary: '{{ $labels.name }} is stopped on {{ $labels.host }}'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: '(windows_service_state{state="running"} == bool 0) and on(instance, name) (windows_service_start_mode{start_mode="auto"} == 1)'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: gt, params: [0]}}]

      - uid: win-disk-full
        title: Windows disk above 85%
        condition: C
        for: 10m
        noDataState: OK
        execErrState: Error
        labels: {severity: warning}
        annotations:
          summary: 'Drive {{ $labels.volume }} on {{ $labels.host }} is {{ printf "%.0f" $values.A.Value }}% full'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: '100 * (1 - windows_logical_disk_free_bytes{volume=~"[A-Z]:"} / windows_logical_disk_size_bytes{volume=~"[A-Z]:"})'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: gt, params: [85]}}]

      - uid: win-cpu-high
        title: Windows CPU high
        condition: C
        for: 15m
        noDataState: OK
        execErrState: Error
        labels: {severity: warning}
        annotations:
          summary: 'CPU on {{ $labels.host }} at {{ printf "%.0f" $values.A.Value }}% for 15 min'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: '100 - avg by (host) (rate(windows_cpu_time_total{mode="idle"}[5m])) * 100'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: gt, params: [90]}}]

      - uid: win-time-drift
        title: Domain clock out of sync
        condition: C
        for: 10m
        noDataState: OK
        execErrState: Error
        labels: {severity: critical}
        annotations:
          summary: 'Clock on {{ $labels.host }} is {{ printf "%.0f" $values.A.Value }} s off. Kerberos logons fail above 5 min'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: 'abs(windows_time_computed_time_offset_seconds)'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: gt, params: [60]}}]

      - uid: win-ad-replication
        title: AD replication backlog
        condition: C
        for: 30m
        noDataState: OK
        execErrState: Error
        labels: {severity: warning}
        annotations:
          summary: '{{ $labels.host }} has {{ printf "%.0f" $values.A.Value }} pending AD replication syncs for 30 min'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: 'sum by (host) (windows_ad_replication_pending_synchronizations)'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: gt, params: [0]}}]

      - uid: win-sql-log-full
        title: SCCM database log almost full
        condition: C
        for: 15m
        noDataState: OK
        execErrState: Error
        labels: {severity: warning}
        annotations:
          summary: 'SQL log of {{ $labels.database }} on {{ $labels.host }} is {{ printf "%.0f" $values.A.Value }}% used'
        data:
          - refId: A
            relativeTimeRange: {from: 600, to: 0}
            datasourceUid: prometheus
            model: {refId: A, instant: true, expr: 'max by (host, database) (windows_mssql_databases_log_used_percent{database=~"CM_.*"})'}
          - refId: C
            datasourceUid: __expr__
            model:
              refId: C
              type: threshold
              expression: A
              conditions: [{evaluator: {type: gt, params: [90]}}]
EOF
cd /opt/monitoring && docker compose restart grafana

nano /opt/monitoring/prometheus/kuma_key        # paste the key, save
cd /opt/monitoring/prometheus
tr -cd 'A-Za-z0-9_' < kuma_key > kuma_key.tmp && mv kuma_key.tmp kuma_key && chmod 644 kuma_key
curl -s -o /dev/null -w '%{http_code}\n' -u "prometheus:$(cat kuma_key)" http://${KUMA_IP}/metrics

cat >> /opt/monitoring/prometheus/prometheus.yml <<EOF

  - job_name: uptimekuma
    scrape_interval: 60s
    basic_auth:
      username: prometheus
      password_file: kuma_key
    static_configs:
      - targets: ['${KUMA_IP}:80']
        labels:
          host: uptime
EOF
curl -s -X POST localhost:9090/-/reload && echo "reload OK"

mkdir -p /opt/monitoring/grafana/provisioning/dashboards/files/Overview
curl -fsSL -o /opt/monitoring/grafana/provisioning/dashboards/files/Overview/zadran-lab-overview.json \
  https://www.zadran.net/downloads/grafana/dashboards/zadran-lab-overview.json

mkdir -p /root/monitoring-setup && cd /root/monitoring-setup
curl -fsSLO https://www.zadran.net/downloads/grafana/setup-monitoring.sh
curl -fsSL -o lab.env https://www.zadran.net/downloads/grafana/lab.env.example
chmod 600 lab.env
nano lab.env

sed 's#^STACK_DIR=.*#STACK_DIR="/tmp/monitoring-test"#' lab.env > lab-test.env
ENV_FILE=lab-test.env SKIP_DOCKER=1 bash setup-monitoring.sh

bash setup-monitoring.sh