#!/usr/bin/env bash
#
# Homelab monitoring, Part 7: An ISP report card with outages, ping and speed tests
# https://www.zadran.net/guides/homelab-monitoring-7-isp-report-card
# Generated from the guide, last updated 2026-10-13.
#
# NO WARRANTY. This script is provided as is. You are responsible for
# understanding what it does before running it, and for the consequences.
# Take a backup first. See https://www.zadran.net/disclaimer
#
# Destructive steps are left commented out on purpose -- uncomment them
# only once you have understood what they do.
#
# Where the guide writes a file with <<'EOF' and your values are in it,
# this script uses <<EOF so that they expand. Every other $, ` and \
# gets a backslash, so the rest comes out exactly as the guide shows it.

set -euo pipefail

# ---- Set these for your environment ----
# From Part 1
CTID='140'
# pfSense or your router, pinged as reference
PFSENSE_IP='192.168.1.1'
# For the speed-test hours
TIMEZONE='Europe/Berlin'
# Hours of the day, comma-separated
SPEEDTEST_HOURS='8,14,20'
# The alert fires below half of it
CONTRACT_DOWN='250'

pct enter ${CTID}

cd /opt/monitoring
mkdir -p blackbox speedtest

cat > blackbox/blackbox.yml <<'EOF'
modules:
  icmp:
    prober: icmp
    timeout: 5s
    icmp:
      preferred_ip_protocol: ip4
      ip_protocol_fallback: false
EOF

curl -fsSL -o speedtest/Dockerfile https://www.zadran.net/downloads/grafana/speedtest/Dockerfile
curl -fsSL -o speedtest/speedtest_exporter.py https://www.zadran.net/downloads/grafana/speedtest/speedtest_exporter.py
ls -l blackbox speedtest

cd /opt/monitoring
mkdir -p backup
cp compose.override.yaml backup/compose.override.yaml.$(date +%Y%m%d-%H%M%S) 2>/dev/null
[ -f compose.override.yaml ] || echo "services:" > compose.override.yaml
cat >> compose.override.yaml <<EOF

  blackbox:
    image: prom/blackbox-exporter:latest
    container_name: blackbox
    restart: unless-stopped
    user: root
    cap_add:
      - NET_RAW
    command:
      - --config.file=/etc/blackbox/blackbox.yml
    volumes:
      - ./blackbox/blackbox.yml:/etc/blackbox/blackbox.yml:ro

  speedtest:
    build: ./speedtest
    image: local/speedtest-exporter
    container_name: speedtest
    restart: unless-stopped
    environment:
      - TZ=${TIMEZONE}
      - SPEEDTEST_HOURS=${SPEEDTEST_HOURS}
EOF
docker compose config -q && docker compose up -d --build blackbox speedtest

P=/opt/monitoring/prometheus/prometheus.yml
cp $P /opt/monitoring/backup/prometheus.yml.$(date +%Y%m%d-%H%M%S)
cat >> $P <<EOF

  - job_name: isp-ping
    metrics_path: /probe
    params:
      module: [icmp]
    static_configs:
      - targets: ['1.1.1.1', '8.8.8.8', '9.9.9.9']
        labels:
          probe: internet
      - targets: ['${PFSENSE_IP}']
        labels:
          probe: pfsense
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox:9115

  - job_name: speedtest
    scrape_interval: 60s
    static_configs:
      - targets: ['speedtest:9798']
        labels:
          host: speedtest
EOF
docker exec prometheus promtool check config /etc/prometheus/prometheus.yml && curl -fsS -X POST localhost:9090/-/reload && echo reloaded

curl -s -G localhost:9090/api/v1/query --data-urlencode 'query=probe_success{job="isp-ping"}' | grep -o '"instance":"[^"]*"\|"value":\[[^]]*\]' | paste - -
docker logs speedtest 2>&1 | tail -2

D=${CONTRACT_DOWN}
curl -fsSL https://www.zadran.net/downloads/grafana/alerting/isp.yaml | sed -e "s/777 Mbit/$D Mbit/g; s/388\.5\]/$((D/2))]/g" > /opt/monitoring/grafana/provisioning/alerting/isp.yaml
grep -n 'Mbit\|params: \[' /opt/monitoring/grafana/provisioning/alerting/isp.yaml
cd /opt/monitoring && docker compose restart grafana