#!/usr/bin/env bash
#
# Homelab monitoring, Part 1: Prometheus and Grafana on Proxmox
# https://www.zadran.net/guides/homelab-monitoring-1-prometheus-grafana
# Generated from the guide, last updated 2026-10-13.
#
# NO WARRANTY. This script is provided as is. You are responsible for
# understanding what it does before running it, and for the consequences.
# Take a backup first. See https://www.zadran.net/disclaimer
#
# Destructive steps are left commented out on purpose -- uncomment them
# only once you have understood what they do.
#
# Where the guide writes a file with <<'EOF' and your values are in it,
# this script uses <<EOF so that they expand. Every other $, ` and \
# gets a backslash, so the rest comes out exactly as the guide shows it.

set -euo pipefail

# ---- Set these for your environment ----
# The IP you give the new container
MON_IP='192.168.1.40'
# Any number not already used on that node
CTID='140'
# Usually your router
GATEWAY='192.168.1.1'
# The IP of your Proxmox server
PVE_IP='192.168.1.2'
# How it is labelled on dashboards and alerts
PVE_NAME='pve'
# Affects graph times
TIMEZONE='Europe/Berlin'
# e.g. 180d, 365d or 730d
RETENTION='365d'

pveam update
TPL=$(pveam available --section system | awk '/debian-13-standard.*_amd64/ {print $2}' | sort -V | tail -1)
echo "Downloading: $TPL"
pveam download local "$TPL"

TPL=$(ls /var/lib/vz/template/cache/ | grep 'debian-13-standard.*_amd64' | sort -V | tail -1)

pct create ${CTID} local:vztmpl/$TPL \
  --hostname monitoring \
  --cores 2 --memory 2048 --swap 512 \
  --rootfs local-lvm:20 \
  --net0 name=eth0,bridge=vmbr0,ip=${MON_IP}/24,gw=${GATEWAY} \
  --nameserver ${GATEWAY} \
  --unprivileged 1 --features nesting=1,keyctl=1 \
  --onboot 1

pct start ${CTID}
pct status ${CTID}
pct exec ${CTID} -- passwd

pct exec ${CTID} -- bash -c "apt update && apt full-upgrade -y && apt install -y curl jq ca-certificates && curl -fsSL https://get.docker.com | sh"
pct exec ${CTID} -- docker version --format '{{.Server.Version}}'

pct enter ${CTID}

mkdir -p /opt/monitoring
nano /opt/monitoring/.env

chmod 600 /opt/monitoring/.env

mkdir -p /opt/monitoring/prometheus /opt/monitoring/grafana/provisioning/{datasources,dashboards,alerting,plugins}

cat > /opt/monitoring/compose.yaml <<EOF
services:
  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    restart: unless-stopped
    command:
      - --config.file=/etc/prometheus/prometheus.yml
      - --storage.tsdb.path=/prometheus
      - --storage.tsdb.retention.time=${RETENTION}
      - --web.enable-lifecycle
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus:/etc/prometheus:ro
      - prometheus-data:/prometheus

  grafana:
    image: grafana/grafana-oss:latest
    container_name: grafana
    restart: unless-stopped
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=\${GRAFANA_ADMIN_PASSWORD}
      - GF_USERS_ALLOW_SIGN_UP=false
      - GF_PLUGINS_PREINSTALL_DISABLED=true
      - TZ=${TIMEZONE}
    volumes:
      - grafana-data:/var/lib/grafana
      - ./grafana/provisioning:/etc/grafana/provisioning:ro
    depends_on:
      - prometheus

volumes:
  prometheus-data:
  grafana-data:
EOF

cat > /opt/monitoring/prometheus/prometheus.yml <<EOF
global:
  scrape_interval: 30s
  evaluation_interval: 30s

scrape_configs:
  - job_name: prometheus
    static_configs:
      - targets: ['localhost:9090']

  - job_name: node
    static_configs:
      - targets: ['${PVE_IP}:9100']
        labels:
          host: ${PVE_NAME}
EOF

cat > /opt/monitoring/grafana/provisioning/datasources/prometheus.yml <<'EOF'
apiVersion: 1
datasources:
  - name: Prometheus
    type: prometheus
    uid: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
EOF

cd /opt/monitoring && docker compose up -d
sleep 20
docker ps --format '{{.Names}}  {{.Status}}'

apt update && apt install -y prometheus-node-exporter
systemctl enable --now prometheus-node-exporter
curl -s localhost:9100/metrics | grep -c '^node_'