This is the live public configuration for the Geomys Tuscolo CT Log Server, a Sunlight instance.
See also the public playbooks.
homehtml: /etc/sunlight/home.html
listen:
- "185.230.223.193:443"
- "[2a0c:2f07:c1::c1]:443"
acme:
hosts:
- tuscolo.sunlight.geomys.org
cache: /var/db/sunlight/autocert/
checkpoints: /tank/shared/checkpoints.db
logs:
- shortname: tuscolo2026h2
inception: 2025-04-27
period: 200
submissionprefix: https://tuscolo2026h2.sunlight.geomys.org
monitoringprefix: https://tuscolo2026h2.skylight.geomys.org
secret: /tank/enc/tuscolo2026h2.seed.bin
cache: /tank/caches/tuscolo2026h2/cache.db
poolsize: 150
localdirectory: /tank/logs/tuscolo2026h2/data
notafterstart: 2026-07-01T00:00:00Z
notafterlimit: 2027-01-01T00:00:00Z
- shortname: tuscolo2027h1
inception: 2025-06-02
period: 200
submissionprefix: https://tuscolo2027h1.sunlight.geomys.org
monitoringprefix: https://tuscolo2027h1.skylight.geomys.org
secret: /tank/enc/tuscolo2027h1.seed.bin
cache: /tank/caches/tuscolo2027h1/cache.db
poolsize: 150
localdirectory: /tank/logs/tuscolo2027h1/data
notafterstart: 2027-01-01T00:00:00Z
notafterlimit: 2027-07-01T00:00:00Z
- shortname: tuscolo2027h2
inception: 2025-06-02
period: 200
submissionprefix: https://tuscolo2027h2.sunlight.geomys.org
monitoringprefix: https://tuscolo2027h2.skylight.geomys.org
secret: /tank/enc/tuscolo2027h2.seed.bin
cache: /tank/caches/tuscolo2027h2/cache.db
poolsize: 150
localdirectory: /tank/logs/tuscolo2027h2/data
notafterstart: 2027-07-01T00:00:00Z
notafterlimit: 2028-01-01T00:00:00Z
- shortname: tuscolo2028h1
inception: 2026-07-28
period: 200
submissionprefix: https://tuscolo2028h1.sunlight.geomys.org
monitoringprefix: https://tuscolo2028h1.skylight.geomys.org
secret: /tank/enc/tuscolo2028h1.seed.bin
cache: /tank/caches/tuscolo2028h1/cache.db
poolsize: 150
localdirectory: /tank/logs/tuscolo2028h1/data
notafterstart: 2028-01-01T00:00:00Z
notafterlimit: 2028-07-01T00:00:00Z
- shortname: tuscolo2028h2
inception: 2026-07-28
period: 200
submissionprefix: https://tuscolo2028h2.sunlight.geomys.org
monitoringprefix: https://tuscolo2028h2.skylight.geomys.org
secret: /tank/enc/tuscolo2028h2.seed.bin
cache: /tank/caches/tuscolo2028h2/cache.db
poolsize: 150
localdirectory: /tank/logs/tuscolo2028h2/data
notafterstart: 2028-07-01T00:00:00Z
notafterlimit: 2029-01-01T00:00:00Z
listen:
- "185.230.223.195:443"
- "[2a0c:2f07:c1::c3]:443"
acme:
hosts:
- navigli.sunlight.geomys.org
cache: /var/db/sunlight/autocert-staging/
checkpoints: /tank/shared/checkpoints.db
witness:
name: witness.navigli.sunlight.geomys.org
mirrorname: oid/1.3.6.1.4.1.66252.128.0
submissionprefix: https://witness.navigli.sunlight.geomys.org
monitoringprefix: https://witness.navigli.skylight.geomys.org
secret: /tank/enc/navigli-witness.seed.bin
localdirectory: /tank/witness-staging
loglists:
- "https://testing.witness-network.org/log-list.1"
- "https://staging.witness-network.org/log-list-10qps-4klogs.1"
- "https://staging.witness-network.org/log-list-100qps-40klogs.1"
- "https://uptime.geomys.org/witness/log-list"
mirrorloglists:
- "https://github.com/geomys/magnolia/raw/refs/heads/main/staging/mirror.log-list.txt"
logs:
- shortname: navigli2026h2
inception: 2025-05-03
period: 200
submissionprefix: https://navigli2026h2.sunlight.geomys.org
monitoringprefix: https://navigli2026h2.skylight.geomys.org
ccadbroots: testing
extraroots: /etc/sunlight/extra-roots-staging.pem
secret: /tank/enc/navigli2026h2.seed.bin
cache: /tank/caches/navigli2026h2/cache.db
poolsize: 150
localdirectory: /tank/logs/navigli2026h2/data
notafterstart: 2026-07-01T00:00:00Z
notafterlimit: 2027-01-01T00:00:00Z
- shortname: navigli2027h1
inception: 2025-06-02
period: 200
submissionprefix: https://navigli2027h1.sunlight.geomys.org
monitoringprefix: https://navigli2027h1.skylight.geomys.org
ccadbroots: testing
extraroots: /etc/sunlight/extra-roots-staging.pem
secret: /tank/enc/navigli2027h1.seed.bin
cache: /tank/caches/navigli2027h1/cache.db
poolsize: 150
localdirectory: /tank/logs/navigli2027h1/data
notafterstart: 2027-01-01T00:00:00Z
notafterlimit: 2027-07-01T00:00:00Z
- shortname: navigli2027h2
inception: 2025-06-02
period: 200
submissionprefix: https://navigli2027h2.sunlight.geomys.org
monitoringprefix: https://navigli2027h2.skylight.geomys.org
ccadbroots: testing
extraroots: /etc/sunlight/extra-roots-staging.pem
secret: /tank/enc/navigli2027h2.seed.bin
cache: /tank/caches/navigli2027h2/cache.db
poolsize: 150
localdirectory: /tank/logs/navigli2027h2/data
notafterstart: 2027-07-01T00:00:00Z
notafterlimit: 2028-01-01T00:00:00Z
- shortname: navigli2028h1
inception: 2026-07-28
period: 200
submissionprefix: https://navigli2028h1.sunlight.geomys.org
monitoringprefix: https://navigli2028h1.skylight.geomys.org
ccadbroots: testing
extraroots: /etc/sunlight/extra-roots-staging.pem
secret: /tank/enc/navigli2028h1.seed.bin
cache: /tank/caches/navigli2028h1/cache.db
poolsize: 150
localdirectory: /tank/logs/navigli2028h1/data
notafterstart: 2028-01-01T00:00:00Z
notafterlimit: 2028-07-01T00:00:00Z
- shortname: navigli2028h2
inception: 2026-07-28
period: 200
submissionprefix: https://navigli2028h2.sunlight.geomys.org
monitoringprefix: https://navigli2028h2.skylight.geomys.org
ccadbroots: testing
extraroots: /etc/sunlight/extra-roots-staging.pem
secret: /tank/enc/navigli2028h2.seed.bin
cache: /tank/caches/navigli2028h2/cache.db
poolsize: 150
localdirectory: /tank/logs/navigli2028h2/data
notafterstart: 2028-07-01T00:00:00Z
notafterlimit: 2029-01-01T00:00:00Z
listen:
- "185.230.223.194:443"
- "[2a0c:2f07:c1::c2]:443"
acme:
hosts:
- tuscolo.skylight.geomys.org
cache: /var/db/sunlight/skylight/
logsjsonprefix: https://tuscolo.skylight.geomys.org
homeredirect: https://tuscolo.sunlight.geomys.org
operatorname: Geomys
witnesses:
- monitoringprefix: https://witness-golf.skylight.geomys.org
localdirectory: /tank/witness-golf
- monitoringprefix: https://witness.navigli.skylight.geomys.org
localdirectory: /tank/witness-staging
staging: true
logs:
- shortname: tuscolo2025h2
monitoringprefix: https://tuscolo2025h2.skylight.geomys.org
localdirectory: /tank/logs/tuscolo2025h2/data
- shortname: tuscolo2026h1
monitoringprefix: https://tuscolo2026h1.skylight.geomys.org
localdirectory: /tank/logs/tuscolo2026h1/data
- shortname: tuscolo2026h2
monitoringprefix: https://tuscolo2026h2.skylight.geomys.org
localdirectory: /tank/logs/tuscolo2026h2/data
- shortname: tuscolo2027h1
monitoringprefix: https://tuscolo2027h1.skylight.geomys.org
localdirectory: /tank/logs/tuscolo2027h1/data
- shortname: tuscolo2027h2
monitoringprefix: https://tuscolo2027h2.skylight.geomys.org
localdirectory: /tank/logs/tuscolo2027h2/data
- shortname: tuscolo2028h1
monitoringprefix: https://tuscolo2028h1.skylight.geomys.org
localdirectory: /tank/logs/tuscolo2028h1/data
- shortname: tuscolo2028h2
monitoringprefix: https://tuscolo2028h2.skylight.geomys.org
localdirectory: /tank/logs/tuscolo2028h2/data
- shortname: navigli2026h1
monitoringprefix: https://navigli2026h1.skylight.geomys.org
localdirectory: /tank/logs/navigli2026h1/data
staging: true
- shortname: navigli2026h2
monitoringprefix: https://navigli2026h2.skylight.geomys.org
localdirectory: /tank/logs/navigli2026h2/data
staging: true
- shortname: navigli2027h1
monitoringprefix: https://navigli2027h1.skylight.geomys.org
localdirectory: /tank/logs/navigli2027h1/data
staging: true
- shortname: navigli2027h2
monitoringprefix: https://navigli2027h2.skylight.geomys.org
localdirectory: /tank/logs/navigli2027h2/data
staging: true
- shortname: navigli2028h1
monitoringprefix: https://navigli2028h1.skylight.geomys.org
localdirectory: /tank/logs/navigli2028h1/data
staging: true
- shortname: navigli2028h2
monitoringprefix: https://navigli2028h2.skylight.geomys.org
localdirectory: /tank/logs/navigli2028h2/data
staging: true
#!/bin/bash
set -euo pipefail
unit_flag="skylight"
display_help() {
echo "Usage: debug [-u unit] {useragents|useragents-bytes|ips|ips-bytes|sourcelimit|keylog={on|off}|logs={on|off}|port}"
}
while getopts "u:h" opt; do
case ${opt} in
u )
unit_flag=$OPTARG
;;
h )
display_help >&2
exit 0
;;
\? )
echo "Invalid option: -$OPTARG" >&2
display_help >&2
exit 1
;;
: )
echo "Option -$OPTARG requires an argument" >&2
display_help >&2
exit 1
;;
esac
done
shift $((OPTIND - 1))
if [ "$#" -ne 1 ]; then
echo "Exactly one positional argument is required" >&2
display_help >&2
exit 1
fi
PID=$(systemctl show "$unit_flag" --property MainPID | cut -d'=' -f2)
if [ -z "$PID" ] || [ "$PID" = "0" ]; then
echo "Unit $unit_flag is not running" >&2
exit 1
fi
PORT=$(ss -tulnp | grep "pid=$PID," | awk '{print $5}' | grep 127.0.0.1) || true
if [ -z "$PORT" ]; then
echo "No port found for unit $unit_flag" >&2
exit 1
fi
case $1 in
useragents )
curl -s "$PORT/debug/heavyhitter/useragents"
;;
useragents-bytes )
curl -s "$PORT/debug/heavyhitter/useragents-bytes"
;;
ips-bytes )
curl -s "$PORT/debug/heavyhitter/ips-bytes"
;;
ips )
curl -s "$PORT/debug/heavyhitter/ips"
;;
sourcelimit )
curl -s "$PORT/debug/sourcelimit"
;;
keylog=on )
curl -s -X POST "$PORT/debug/keylog/on"
;;
keylog=off )
curl -s -X POST "$PORT/debug/keylog/off"
;;
logs=on )
curl -s -X POST "$PORT/debug/logs/on"
;;
logs=off )
curl -s -X POST "$PORT/debug/logs/off"
;;
port )
echo "$PORT"
;;
* )
echo "Invalid argument: $1" >&2
display_help >&2
exit 1
;;
esac
[Unit]
Description=Sunlight Certificate Transparency Log
After=network-online.target tank-enc.mount
Wants=network-online.target
StartLimitIntervalSec=0
[Service]
ExecStart=/usr/local/bin/sunlight -c /etc/sunlight/sunlight.yaml
ExecReload=kill -HUP $MAINPID
StandardOutput=append:/var/log/sunlight.jsonl
StandardError=journal
Restart=always
RestartSteps=20
RestartSec=1ms
RestartMaxDelaySec=10s
LimitMEMLOCK=256M
MemorySwapMax=0
[Install]
WantedBy=tank-enc.mount
[Unit]
Description=Sunlight Certificate Transparency Log (staging)
After=network-online.target tank-enc.mount
Wants=network-online.target
StartLimitIntervalSec=0
[Service]
ExecStart=/usr/local/bin/sunlight-staging -c /etc/sunlight/sunlight-staging.yaml
ExecReload=kill -HUP $MAINPID
StandardOutput=append:/var/log/sunlight-staging.jsonl
StandardError=journal
Restart=always
RestartSteps=20
RestartSec=1ms
RestartMaxDelaySec=10s
MemoryMax=8G
MemorySwapMax=0
RuntimeMaxSec=1d
LimitMEMLOCK=256M
[Install]
WantedBy=tank-enc.mount
[Unit]
Description=Sunlight Certificate Transparency Log (read path)
After=network-online.target
Wants=network-online.target
StartLimitIntervalSec=0
[Service]
ExecStart=/usr/local/bin/skylight -c /etc/sunlight/skylight.yaml
StandardOutput=append:/var/log/skylight.jsonl
StandardError=journal
Restart=always
RestartSteps=20
RestartSec=1ms
RestartMaxDelaySec=10s
LimitMEMLOCK=256M
MemorySwapMax=0
[Install]
WantedBy=multi-user.target
[Unit]
Description=Clean up partial tiles
[Service]
Type=oneshot
ExecStart=/usr/local/bin/partial-aftersun -c /etc/sunlight/sunlight.yaml -metrics /var/lib/prometheus/node-exporter/partial-aftersun.prom
ExecStartPost=/usr/bin/curl --retry 3 --retry-delay 1 -m 15 https://sm.hetrixtools.net/hb/?s=a4f010ea1bd8d93598fc96f94000190f
StandardOutput=append:/var/log/partial-aftersun.jsonl
StandardError=journal
[Unit]
Description=Periodically run partial tiles cleanup while Sunlight is running
RefuseManualStart=yes
PartOf=sunlight.service
[Timer]
OnActiveSec=5s
OnUnitActiveSec=5m
[Install]
WantedBy=sunlight.service
[Unit]
Description=Clean up partial tiles (staging)
[Service]
Type=oneshot
ExecStart=/usr/local/bin/partial-aftersun -c /etc/sunlight/sunlight-staging.yaml -metrics /var/lib/prometheus/node-exporter/partial-aftersun-staging.prom
ExecStartPost=/usr/bin/curl --retry 3 --retry-delay 1 -m 15 https://sm.hetrixtools.net/hb/?s=590ce0eb9954e649e6f2be05a4c651bd
StandardOutput=append:/var/log/partial-aftersun-staging.jsonl
StandardError=journal
[Unit]
Description=Periodically run partial tiles cleanup while Sunlight is running (staging)
RefuseManualStart=yes
PartOf=sunlight-staging.service
[Timer]
OnActiveSec=5s
OnUnitActiveSec=5m
[Install]
WantedBy=sunlight-staging.service
[Unit]
Description=https://github.com/FiloSottile/mostly-harmless/tree/main/public-config
After=network-online.target
Wants=network-online.target
Before=caddy.service
StartLimitIntervalSec=0
[Service]
ExecStart=/usr/local/bin/public-config -name tuscolo
DynamicUser=true
Restart=always
RestartSteps=20
RestartSec=1ms
RestartMaxDelaySec=10s
[Install]
WantedBy=multi-user.target
/var/log/*.jsonl {
daily
rotate 10
copytruncate
compress
delaycompress
notifempty
missingok
}
skylight.geomys.org:443 {
bind 185.230.223.196 [2a0c:2f07:c1::c4]
redir https://tuscolo.skylight.geomys.org{uri} permanent
}
config.tuscolo.sunlight.geomys.org:443 {
bind 185.230.223.196 [2a0c:2f07:c1::c4]
reverse_proxy localhost:8080
}
config.sunlight.geomys.org:443 {
bind 185.230.223.196 [2a0c:2f07:c1::c4]
redir https://config.tuscolo.sunlight.geomys.org{uri} permanent
}
stats.tuscolo.sunlight.geomys.org:443 {
bind 185.230.223.196 [2a0c:2f07:c1::c4]
root * /var/www/heliograph
file_server
encode zstd gzip
header Cache-Control "public, max-age=60"
}
stats.sunlight.geomys.org:443 {
bind 185.230.223.196 [2a0c:2f07:c1::c4]
redir https://stats.tuscolo.sunlight.geomys.org{uri} permanent
}
stats.trastevere.sunlight.geomys.org:443 {
bind 185.230.223.196 [2a0c:2f07:c1::c4]
root * /var/www/heliograph-trastevere
file_server
encode zstd gzip
header Cache-Control "public, max-age=60"
}
alerts.sunlight.geomys.org:443 {
bind 185.230.223.196 [2a0c:2f07:c1::c4]
@get method GET HEAD
handle @get {
rewrite * /api/v1/query?query=label_replace%28vector%281%29%2C%20%22status%22%2C%20%22no_pages_firing%22%2C%20%22%22%2C%20%22%22%29%20unless%20on%28%29%20ALERTS%7Balertstate%3D%22firing%22%2Cseverity%3D%22page%22%7D
reverse_proxy localhost:9090
}
respond 405
}
keyserver.geomys.org:443 {
bind 185.230.223.196 [2a0c:2f07:c1::c4]
reverse_proxy localhost:13889
}
pkg.geomys.dev:443 {
bind 185.230.223.196 [2a0c:2f07:c1::c4]
reverse_proxy localhost:8081
}
plc.geomys.org:443 {
bind 185.230.223.196 [2a0c:2f07:c1::c4]
reverse_proxy localhost:6780
}
[Unit]
StartLimitIntervalSec=0
[Service]
Restart=always
RestartSteps=20
RestartSec=1ms
RestartMaxDelaySec=10s
#!/bin/sh
set -eu
out=/var/lib/prometheus/node-exporter/zfs.prom
tmp=$(mktemp "$out.XXXXXX")
trap 'rm -f "$tmp"' EXIT
{
echo '# TYPE zfs_dataset_referenced_bytes gauge'
echo '# TYPE zfs_dataset_logicalreferenced_bytes gauge'
echo '# TYPE zfs_dataset_available_bytes gauge'
zfs list -Hp -r -o name,referenced,logicalreferenced,available tank/logs tank/caches \
| awk '{
printf "zfs_dataset_referenced_bytes{dataset=\"%s\"} %s\n", $1, $2
printf "zfs_dataset_logicalreferenced_bytes{dataset=\"%s\"} %s\n", $1, $3
printf "zfs_dataset_available_bytes{dataset=\"%s\"} %s\n", $1, $4
}'
echo '# HELP zfs_txg_last_sync_seconds Sync duration of the most recent committed transaction group.'
echo '# TYPE zfs_txg_last_sync_seconds gauge'
echo '# HELP zfs_txg_max_sync_seconds Maximum txg sync duration over the kernel history ring.'
echo '# TYPE zfs_txg_max_sync_seconds gauge'
echo '# HELP zfs_txg_last_dirty_bytes Dirty data in the most recent committed transaction group.'
echo '# TYPE zfs_txg_last_dirty_bytes gauge'
for txgs in /proc/spl/kstat/zfs/*/txgs; do
pool=$(basename "$(dirname "$txgs")")
awk -v pool="$pool" '
$3 == "C" { last_sync = $12; if ($12 > max_sync) max_sync = $12; last_dirty = $4 }
END {
if (last_sync != "") {
printf "zfs_txg_last_sync_seconds{pool=\"%s\"} %.6f\n", pool, last_sync / 1e9
printf "zfs_txg_max_sync_seconds{pool=\"%s\"} %.6f\n", pool, max_sync / 1e9
printf "zfs_txg_last_dirty_bytes{pool=\"%s\"} %d\n", pool, last_dirty
}
}' "$txgs"
done
# Error counters are cumulative since the last zpool clear or reboot.
zpool_status=$(zpool status -j --json-int --json-flat-vdevs)
printf '%s\n' "$zpool_status" | jq -r '
"# HELP zfs_pool_scan_active Whether a scrub or resilver is in progress.",
"# TYPE zfs_pool_scan_active gauge",
(.pools[] | "zfs_pool_scan_active{pool=\"\(.name)\"} \(if .scan_stats.state == "SCANNING" then 1 else 0 end)"),
"# HELP zfs_pool_data_errors Permanent data errors reported by zpool status.",
"# TYPE zfs_pool_data_errors gauge",
(.pools[] | "zfs_pool_data_errors{pool=\"\(.name)\"} \(.error_count // 0)"),
"# HELP zfs_vdev_errors Read, write and checksum errors since the last zpool clear.",
"# TYPE zfs_vdev_errors gauge",
(.pools[] | .name as $pool | .vdevs[] | select(has("read_errors")) |
"zfs_vdev_errors{pool=\"\($pool)\",vdev=\"\(.name)\",type=\"read\"} \(.read_errors)",
"zfs_vdev_errors{pool=\"\($pool)\",vdev=\"\(.name)\",type=\"write\"} \(.write_errors)",
"zfs_vdev_errors{pool=\"\($pool)\",vdev=\"\(.name)\",type=\"checksum\"} \(.checksum_errors)")'
# Slab memory (dentries, inodes) is charged to the cgroup that created it,
# and moves to system.slice itself when a transient unit's cgroup goes away,
# where the ZFS shrinker can't see it. Units can vanish between the glob and
# the read, hence the tolerant awk.
echo '# HELP cgroup_memory_stat_bytes Selected memory.stat entries of a cgroup, including its descendants.'
echo '# TYPE cgroup_memory_stat_bytes gauge'
for d in /sys/fs/cgroup/system.slice /sys/fs/cgroup/system.slice/*/; do
d=${d%/}
awk -v cg="${d#/sys/fs/cgroup/}" '
$1 ~ /^(anon|file|slab_reclaimable|slab_unreclaimable)$/ {
printf "cgroup_memory_stat_bytes{cgroup=\"%s\",stat=\"%s\"} %s\n", cg, $1, $2
}' "$d/memory.stat" 2>/dev/null || true
done
} > "$tmp"
chmod 0644 "$tmp"
mv "$tmp" "$out"
[Unit]
Description=Write ZFS dataset metrics for node_exporter textfile collector
[Service]
Type=oneshot
ExecStart=/usr/local/bin/zfs-textfile
TimeoutStartSec=30s
[Unit]
Description=Run zfs-textfile every minute
[Timer]
OnBootSec=30s
OnUnitActiveSec=60s
AccuracySec=1s
[Install]
WantedBy=timers.target
# Set the command-line arguments to pass to the server.
# Due to shell escaping, to pass backslashes for regexes, you need to double
# them (\\d for \d). If running under systemd, you need to double them again
# (\\\\d to mean \d), and escape newlines too.
ARGS="--collector.vmstat.fields=^(oom_kill|pgpg|pswp|pg.*fault|pgsteal_|pgscan_|slabs_scanned|kswapd_|pginodesteal|allocstall_|drop_pagecache|drop_slab).* --collector.slabinfo --collector.sysctl --collector.sysctl.include=vm.min_free_kbytes --collector.sysctl.include=vm.watermark_scale_factor --collector.sysctl.include=vm.swappiness --collector.sysctl.include=vm.vfs_cache_pressure --collector.sysctl.include=fs.dentry-state:nr_dentry,nr_unused,age_limit,want_pages,nr_negative,dummy --collector.sysctl.include=fs.inode-nr:nr_inodes,nr_free_inodes"
[Service]
ExecStartPre=+/bin/chgrp prometheus /proc/slabinfo
ExecStartPre=+/bin/chmod 0440 /proc/slabinfo
# Drops the reclaimable memory charged to a service's cgroup. This bounds the
# dentries, and through them the ZFS znodes, dnodes and ARC metadata.
#
# It works around a combination of OpenZFS (2.3) and cgroup behaviors that can
# cause reads to stall for ~2 ms.
#
# 1. Under pressure, ARC allocations are throttled.
#
# Every ZFS read and write allocates an ARC buffer (arc_get_data_impl),
# whether or not the data is worth caching. If the ARC is more than a small
# margin above its target size c, each allocation waits for an eviction pass
# (arc_wait_for_eviction). If the pass finds nothing evictable, the waiter is
# released anyway, having paid for a full futile scan.
#
# The ARC is not needed for the log to perform (cache.db reads cost 0.25 ms
# with the ARC nearly empty of data) but there is no way to opt out of the
# wait. (direct=always would bypass it, but SQLite's page buffers are not
# page-aligned, so it would never engage.)
#
# 2. The target ignores unevictable memory.
#
# Under memory pressure, arc_reduce_target_size lowers c with no floor other
# than c_min (4 GiB here), regardless of how much of the ARC is pinned by
# open dnodes and their metadata. Once the pinned set alone exceeds c, the
# ARC is permanently over target and every read is throttled as in (1).
#
# 3. A lot of memory is unevictable for ZFS due to cgroups.
#
# Dentries are charged to the memory cgroup of the process that looked them
# up (e.g. skylight's for the tiles it serves). Each dentry pins a znode,
# which pins a dnode and ~5 KiB of ARC metadata (dnode block, dbuf, bonus
# buffer). The only way ZFS has to release them is arc_prune -> zfs_prune ->
# super_cache_scan, which scans the root memory cgroup only. So for ZFS this
# memory is unevictable.
#
# 4. Memory pressure is handled by ZFS, never by the kernel.
#
# The kernel could evict the dentries charged to all cgroups, but ZFS starts
# shrinking its target when free plus inactive file memory drops under
# zfs_arc_sys_free (4.25 GiB here), while kswapd only wakes under the zone
# low watermark (~200 MiB here). The kernel's reclaim rarely runs, and when
# it does it stops as soon as its watermark is met, a few hundred MiB later.
#
# The result is that ZFS is the only one trying to reclaim memory (4), and while
# it can't succeed (3) it keeps lowering its target (2), which throttles every
# read (1).
#
# Dropping the dentries through memory.reclaim leaves the znodes on the root
# inode LRU, where ZFS pruning (and kswapd) can dispose of them, turning the
# pinned metadata back into ordinary evictable ARC.
[Unit]
Description=Reclaim page cache and slab charged to %i.service
[Service]
Type=oneshot
ExecStart=/usr/local/bin/memcg-reclaim %i
TimeoutStartSec=5m
# Hourly bound pinned ZFS metadata (dentries, and through them the ZFS znodes,
# dnodes and ARC metadata) that skylight accumulates by serving tiles at roughly
# an hour of tile lookups (~0.4M dentries, ~2 GiB of ARC).
#
# The cost is that the next lookup of each tile re-reads its dnode.
#
# See memcg-reclaim@.service.
[Unit]
Description=Hourly reclaim of memory charged to skylight.service
PartOf=skylight.service
[Timer]
OnActiveSec=1h
OnUnitActiveSec=1h
AccuracySec=1m
[Install]
WantedBy=skylight.service
#!/bin/sh
# Usage: memcg-reclaim UNIT
#
# Drops the reclaimable memory (page cache, dentries, inodes) charged to a
# systemd service's memory cgroup. See memcg-reclaim@.service.
set -eu
cg=/sys/fs/cgroup/system.slice/$1.service
mstat() { awk -v k="$1" '$1 == k { print $2 }' "$cg/memory.stat"; }
before=$(mstat slab_reclaimable)
# Ask for every file page and reclaimable slab byte charged to the cgroup.
# The kernel reclaims what it can and then reports EAGAIN if it could not
# reach the requested amount (partially freed slabs are not page-freeable),
# so a failed write is the normal outcome and the effect is checked below.
# swappiness=0 keeps the pass away from anonymous memory, which could not
# be swapped anyway (MemorySwapMax=0).
echo "$(( $(mstat file) + before )) swappiness=0" > "$cg/memory.reclaim" 2>/dev/null || true
after=$(mstat slab_reclaimable)
echo "$1: slab_reclaimable $((before >> 20)) MiB -> $((after >> 20)) MiB;" \
"$(awk '{ print $1 }' /proc/sys/fs/dentry-state) dentries system-wide"
# Fail loudly (node_systemd_unit_state) if the reclaim had no effect, e.g.
# because the objects are actually in use or the cgroup layout changed.
[ "$after" -lt $((256 << 20)) ]
listen:
- "185.230.223.197:443"
- "[2a0c:2f07:c1::c5]:443"
acme:
cache: /var/db/sunlight/autocert-golf/
checkpoints: /tank/shared/checkpoints.db
witness:
name: witness-golf.sunlight.geomys.org
monitoringprefix: https://witness-golf.skylight.geomys.org
secret: /tank/enc/golf-witness.seed.bin
localdirectory: /tank/witness-golf
loglists:
- "https://geomys-loglistfilter.exe.xyz/filter?url=https%3A%2F%2Fwitnessplz.transparency.goog%2Flog-list-10qps-10klogs.txt"
- "https://uptime.geomys.org/witness/log-list"
[Unit]
Description=Sunlight Witness (golf)
After=network-online.target tank-enc.mount
Wants=network-online.target
StartLimitIntervalSec=0
[Service]
ExecStart=/usr/local/bin/sunlight-golf -c /etc/sunlight/sunlight-golf.yaml
ExecReload=kill -HUP $MAINPID
StandardOutput=append:/var/log/sunlight-golf.jsonl
StandardError=journal
Restart=always
RestartSteps=20
RestartSec=1ms
RestartMaxDelaySec=10s
LimitMEMLOCK=256M
MemorySwapMax=0
[Install]
WantedBy=tank-enc.mount
[Unit]
Description=Generate heliograph CT log dashboard
After=prometheus.service
[Service]
Type=oneshot
ExecStart=/usr/local/bin/heliograph-dashboard \
-prometheus http://localhost:9090 \
-title "Tuscolo CT log" \
-log-name tuscolo \
-o /var/www/heliograph/index.html
TimeoutStartSec=30s
ExecStartPost=/usr/bin/curl --retry 3 --retry-delay 1 -m 15 https://sm.hetrixtools.net/hb/?s=32f9e49e38582dfcf7c17c09dc3df085
ProtectSystem=strict
ReadWritePaths=/var/www/heliograph
ProtectHome=yes
NoNewPrivileges=yes
PrivateTmp=yes
PrivateDevices=yes
RestrictAddressFamilies=AF_INET AF_INET6
[Unit]
Description=Refresh heliograph dashboard every minute
[Timer]
OnBootSec=20s
OnUnitActiveSec=5m
AccuracySec=5s
[Install]
WantedBy=timers.target
[Unit]
Description=Generate heliograph CT log dashboard (Trastevere)
After=prometheus.service
[Service]
Type=oneshot
ExecStart=/usr/local/bin/heliograph-dashboard \
-prometheus http://localhost:9090 \
-title "Trastevere CT log" \
-log-name trastevere -node-job trastevere-node \
-network-device 'ens2f.*' -skylight-job trastevere-skylight \
-o /var/www/heliograph-trastevere/index.html
TimeoutStartSec=30s
ExecStartPost=/usr/bin/curl --retry 3 --retry-delay 1 -m 15 https://sm.hetrixtools.net/hb/?s=3ad11212eb47a3a999add43e6d43aa62
ProtectSystem=strict
ReadWritePaths=/var/www/heliograph-trastevere
ProtectHome=yes
NoNewPrivileges=yes
PrivateTmp=yes
PrivateDevices=yes
RestrictAddressFamilies=AF_INET AF_INET6
[Unit]
Description=Refresh heliograph dashboard every minute (Trastevere)
[Timer]
OnBootSec=20s
OnUnitActiveSec=5m
AccuracySec=5s
[Install]
WantedBy=timers.target
[Unit]
Description=Email-authenticated age public key server
After=network-online.target tank-enc.mount
Wants=network-online.target
StartLimitIntervalSec=0
[Service]
Type=exec
ExecStart=/usr/local/bin/age-keyserver -listen localhost:13889 -db /tank/keyserver/keyserver.sqlite3 -logdir /tank/keyserver/tlog
EnvironmentFile=/tank/enc/age-keyserver.env
StandardOutput=append:/var/log/age-keyserver.jsonl
StandardError=journal
Restart=always
RestartSteps=20
RestartSec=1ms
RestartMaxDelaySec=10s
[Install]
WantedBy=tank-enc.mount
[Unit]
After=network-online.target
Wants=network-online.target
Before=caddy.service
StartLimitIntervalSec=0
[Service]
ExecStart=/usr/local/bin/pkg.geomys.dev -addr localhost:8081
DynamicUser=true
Restart=always
RestartSteps=20
RestartSec=1ms
RestartMaxDelaySec=10s
[Install]
WantedBy=multi-user.target
[Unit]
Description=PLC Directory Replica
Documentation=https://github.com/did-method-plc/go-didplc/tree/main/cmd/plc-replica
After=network-online.target
Wants=network-online.target
Before=caddy.service
StartLimitIntervalSec=0
[Service]
ExecStart=/usr/local/bin/plc-replica \
--db-url "sqlite:///tank/plc/replica.db?mode=rwc&cache=shared&_journal_mode=WAL" \
--bind localhost:6780 \
--metrics-addr localhost:9464 \
--num-workers 16 \
--log-json
StandardOutput=append:/var/log/plc-replica.jsonl
StandardError=journal
Restart=always
RestartSteps=20
RestartSec=1ms
RestartMaxDelaySec=10s
MemoryMax=24G
MemorySwapMax=0
ProtectSystem=strict
ReadWritePaths=/tank/plc
ProtectHome=yes
NoNewPrivileges=yes
PrivateTmp=yes
[Install]
WantedBy=multi-user.target
global:
scrape_interval: 15s
scrape_configs:
- job_name: prometheus
static_configs:
- targets:
- localhost:9090
- job_name: node
static_configs:
- targets:
- localhost:9100
- job_name: plc
static_configs:
- targets:
- localhost:9464
- job_name: tuscolo
scheme: https
static_configs:
- targets:
- tuscolo.sunlight.geomys.org
- job_name: navigli
scheme: https
static_configs:
- targets:
- navigli.sunlight.geomys.org
- job_name: skylight
scheme: https
static_configs:
- targets:
- tuscolo.skylight.geomys.org
- job_name: golf
scheme: https
static_configs:
- targets:
- witness-golf.sunlight.geomys.org
- job_name: trastevere
scheme: https
static_configs:
- targets:
- trastevere.sunlight.geomys.org
- job_name: loreto
scheme: https
static_configs:
- targets:
- loreto.sunlight.geomys.org
- job_name: trastevere-skylight
scheme: https
static_configs:
- targets:
- trastevere.skylight.geomys.org
- job_name: trastevere-node
static_configs:
- targets:
- 213.171.190.242:9100
- job_name: twig
scheme: https
static_configs:
- targets:
- log.twig.ct.letsencrypt.org
- job_name: sycamore
scheme: https
static_configs:
- targets:
- log.sycamore.ct.letsencrypt.org
- job_name: willow
scheme: https
static_configs:
- targets:
- log.willow.ct.letsencrypt.org
- job_name: gouda
scheme: https
static_configs:
- targets:
- gouda2027h2.log.ct.ipng.ch
- job_name: rennet
scheme: https
static_configs:
- targets:
- rennet2027h2.log.ct.ipng.ch
- job_name: ctuptime
scheme: https
metrics_path: /ctuptime/metrics
static_configs:
- targets:
- mcpherrinm.github.io
- job_name: sctdelay
scheme: https
metrics_path: /chrome-sctauditing-delay/metrics.txt
static_configs:
- targets:
- agwa-bot.github.io
- job_name: sslmate
scheme: https
metrics_path: /ct_logs.prom
static_configs:
- targets:
- feeds.sslmate.com
rule_files:
- /etc/prometheus/rules.yml
- /etc/prometheus/alerts.yml
alerting:
alertmanagers:
- scheme: https
basic_auth:
username: prometheus
password_file: /etc/prometheus/geomys-alertmanager.token
static_configs:
- targets: ['geomys-alertmanager.exe.xyz:443']
# Prometheus recording rules required by heliograph-dashboard.
#
# The add-chain counter carries issuer and root labels, so it has thousands of
# series per log. Aggregating it on the fly over the chart window is too slow,
# and gets slower every time a new CA starts submitting. These rules keep the
# aggregation incremental: each evaluation only touches the last five minutes.
#
# Install as /etc/prometheus/rules.yml and reference it from prometheus.yml:
#
# rule_files:
# - /etc/prometheus/rules.yml
#
# To backfill history for a freshly added rule (this Prometheus runs with
# --storage.tsdb.allow-overlapping-blocks):
#
# promtool tsdb create-blocks-from rules --url=http://localhost:9090 \
# --start=$(date -d '-7 days' +%s) --end=$(date +%s) \
# --output-dir=/tmp/backfill /etc/prometheus/rules.yml
# mv /tmp/backfill/* /tank/prometheus/
groups:
- name: heliograph
interval: 30s
rules:
- record: log:sunlight_addchain_requests:rate5m
expr: sum by (job, log) (rate(sunlight_addchain_requests_total{error=""}[5m]))
- record: low_priority:sunlight_addchain_requests:rate5m
expr: sum by (job, low_priority) (rate(sunlight_addchain_requests_total{error=""}[5m]))
# Includes failed requests, unlike the two above. The error label is a
# bounded set of categories, so this stays at a few dozen series per log.
- record: source_error:sunlight_addchain_requests:rate5m
expr: sum by (job, log, source, error) (rate(sunlight_addchain_requests_total[5m]))
# Alerting rules for the Geomys Tuscolo and Trastevere logs.
#
# severity="page" wakes someone up; severity="ticket" is for working hours.
groups:
- name: sunlight-alert-recording
interval: 1m
rules:
- record: job_log:sunlight_cache_get_seconds:avg15m
expr: |
sum by (job, log) (rate(sunlight_cache_get_duration_seconds_sum{job=~"tuscolo|trastevere|navigli|loreto"}[15m]))
/
sum by (job, log) (rate(sunlight_cache_get_duration_seconds_count{job=~"tuscolo|trastevere|navigli|loreto"}[15m]))
- record: job_log:sunlight_cache_get:rate15m
expr: sum by (job, log) (rate(sunlight_cache_get_duration_seconds_count{job=~"tuscolo|trastevere|navigli|loreto"}[15m]))
- record: job_log:sunlight_cache_get:rate5m
expr: sum by (job, log) (rate(sunlight_cache_get_duration_seconds_count{job=~"tuscolo|trastevere|navigli|loreto"}[5m]))
# Fraction of time the serialized admission path spends in dedup lookups.
- record: job_log:sunlight_admission_utilization:rate15m
expr: sum by (job, log) (rate(sunlight_cache_get_duration_seconds_sum{job=~"tuscolo|trastevere|navigli|loreto"}[15m]))
- record: job_log:sunlight_admission_utilization:rate5m
expr: sum by (job, log) (rate(sunlight_cache_get_duration_seconds_sum{job=~"tuscolo|trastevere|navigli|loreto"}[5m]))
- record: job_log:sunlight_cache_put_cost_seconds:avg15m
expr: |
sum by (job, log) (rate(sunlight_cache_put_duration_seconds_sum{job=~"tuscolo|trastevere|navigli|loreto"}[15m]))
/
sum by (job, log) (rate(sunlight_cache_put_entries_total{job=~"tuscolo|trastevere|navigli|loreto"}[15m]))
- record: job_log:sunlight_cache_put_entries:rate15m
expr: sum by (job, log) (rate(sunlight_cache_put_entries_total{job=~"tuscolo|trastevere|navigli|loreto"}[15m]))
- record: job_log:sunlight_addchain_organic_shed:rate1m
expr: |
sum by (job, log) (rate(sunlight_addchain_requests_total{job=~"tuscolo|trastevere|navigli|loreto", low_priority!="true", source=~"ratelimit|evicted|timeout"}[1m]))
- record: job_log_endpoint:sunlight_http_requests:rate5m
expr: sum by (job, log, endpoint) (rate(sunlight_http_requests_total{job=~"tuscolo|trastevere|navigli|loreto", endpoint=~"add-chain|add-pre-chain"}[5m]))
- record: instance:node_zfs_arc_overflow_bytes
expr: node_zfs_arc_size{job=~"node|trastevere-node"} - node_zfs_arc_c{job=~"node|trastevere-node"}
# Share of the ARC target held by unevictable metadata.
- record: instance:node_zfs_arc_pinned_ratio
expr: |
(
node_zfs_arc_metadata_size{job=~"node|trastevere-node"}
- node_zfs_arc_mru_evictable_metadata{job=~"node|trastevere-node"}
- node_zfs_arc_mfu_evictable_metadata{job=~"node|trastevere-node"}
+ node_zfs_arc_dnode_size{job=~"node|trastevere-node"}
+ node_zfs_arc_dbuf_size{job=~"node|trastevere-node"}
+ node_zfs_arc_bonus_size{job=~"node|trastevere-node"}
+ node_zfs_arc_hdr_size{job=~"node|trastevere-node"}
) / node_zfs_arc_c{job=~"node|trastevere-node"}
- record: instance:node_dentry_positive
expr: node_sysctl_fs_dentry_state_nr_dentry{job=~"node|trastevere-node"} - node_sysctl_fs_dentry_state_nr_negative{job=~"node|trastevere-node"}
# 1 if witness golf served a 5xx or a p75 add-checkpoint latency over 1s in the last minute.
- record: golf:witness_slo_bad_minute
expr: |
clamp_max(
((sum(increase(sunlight_witness_http_requests_total{job="golf", endpoint="add-checkpoint", code=~"5.."}[1m])) or vector(0)) > bool 0)
+ ((max(sunlight_witness_http_request_duration_seconds{job="golf", endpoint="add-checkpoint", quantile="0.75"}) or vector(0)) > bool 1)
, 1)
- record: golf:witness_slo_bad_minute_ratio:30d
expr: avg_over_time(golf:witness_slo_bad_minute[30d])
- name: sunlight-log-prod
rules:
- alert: SunlightCheckpointStale
expr: |
(time() - sunlight_tree_timestamp_seconds{job=~"tuscolo|trastevere"} > 30)
and on (job, log)
(sunlight_config_notafter_end_timestamp_seconds{job=~"tuscolo|trastevere"} > time())
for: 1m
labels:
severity: page
annotations:
summary: '{{ $labels.log }} checkpoint is {{ $value | humanizeDuration }} old (MMD is 60s)'
description: Sequencing has stalled. Check the sunlight journal for sequencing errors.
- alert: SunlightSequencingFailed
expr: increase(sunlight_sequencing_rounds_total{job=~"tuscolo|trastevere", error!=""}[10m]) > 0
labels:
severity: page
annotations:
summary: '{{ $labels.log }} had {{ $value | humanize }} failed sequencing rounds in 10m ({{ $labels.error }})'
description: Sequencing rounds are failing. The error label and the sunlight journal say why.
- alert: SunlightSubmissionSaturated
expr: |
(job_log:sunlight_admission_utilization:rate5m{job=~"tuscolo|trastevere"} > 0.8)
and on (job, log)
(job_log:sunlight_cache_get:rate5m > 10)
for: 5m
labels:
severity: page
annotations:
summary: '{{ $labels.log }} admission path is {{ $value | humanizePercentage }} busy'
description: >-
Submissions are queueing behind dedup cache lookups. If lookups are slow
(job_log:sunlight_cache_get_seconds:avg15m), it's host memory/ARC; if not, it's the
arrival rate.
- alert: SunlightSequencerSlow
expr: |
(sunlight_addchain_wait_seconds{job=~"tuscolo|trastevere", quantile="0.9"} > 1)
and on (job, log)
(sum by (job, log) (rate(sunlight_addchain_wait_seconds_count{job=~"tuscolo|trastevere"}[5m])) > 5)
for: 2m
labels:
severity: page
annotations:
summary: '{{ $labels.log }} submissions wait {{ $value | humanizeDuration }} (p90) for sequencing'
description: Sequencing rounds are slow. sequencing_phase_duration_seconds shows which phase.
- alert: SunlightOrganicShedding
expr: job_log:sunlight_addchain_organic_shed:rate1m{job=~"tuscolo|trastevere"} > 0.5
for: 5m
labels:
severity: page
annotations:
summary: '{{ $labels.log }} is rejecting {{ $value | humanize }}/s of non-low-priority submissions'
description: The pool is full or the admission path is saturated.
- alert: SunlightTargetDown
expr: up{job=~"tuscolo|trastevere|skylight|trastevere-skylight|golf"} == 0
for: 2m
labels:
severity: page
annotations:
summary: '{{ $labels.job }} ({{ $labels.instance }}) is down or unreachable'
description: Check the unit's status and journal on the host.
- name: sunlight-log-prod-tickets
rules:
- alert: SunlightSubmissionBusy
expr: |
(job_log:sunlight_admission_utilization:rate15m{job=~"tuscolo|trastevere"} > 0.4)
and on (job, log)
(job_log:sunlight_cache_get:rate15m > 10)
for: 5m
labels:
severity: ticket
annotations:
summary: '{{ $labels.log }} admission path is {{ $value | humanizePercentage }} busy (pages at 80%)'
- alert: SunlightAddChainSlow
expr: |
(sunlight_http_request_duration_seconds{job=~"tuscolo|trastevere", quantile="0.5", endpoint=~"add-chain|add-pre-chain"} > 0.3)
and on (job, log, endpoint)
(job_log_endpoint:sunlight_http_requests:rate5m > 5)
for: 2m
labels:
severity: ticket
annotations:
summary: '{{ $labels.log }} {{ $labels.endpoint }} median latency is {{ $value | humanizeDuration }}'
- alert: SunlightAddChainServerErrors
expr: sum by (job, log, endpoint) (increase(sunlight_http_requests_total{job=~"tuscolo|trastevere", code="500"}[15m])) > 0
labels:
severity: ticket
annotations:
summary: '{{ $labels.log }} {{ $labels.endpoint }} returned {{ $value | humanize }} HTTP 500s in 15m'
- alert: SunlightTreeNotGrowing
expr: |
(increase(sunlight_tree_size_leaves_total{job=~"tuscolo|trastevere"}[1h]) == 0)
and on (job, log)
(sunlight_config_notafter_end_timestamp_seconds{job=~"tuscolo|trastevere"} > time())
labels:
severity: ticket
annotations:
summary: '{{ $labels.log }} has not grown in 1h'
description: Nothing is being accepted, or the uptime test submitter has stopped (if all shards stopped at once).
- alert: SunlightRestartedWithoutDeploy
expr: |
(changes(process_start_time_seconds{job=~"tuscolo|trastevere|golf"}[5m]) > 0)
and on (job)
(count by (job) (count_over_time(sunlight_build_info{job=~"tuscolo|trastevere|golf"}[5m])) == 1)
labels:
severity: ticket
annotations:
summary: '{{ $labels.job }} restarted without a new build'
- alert: SkylightRestartedWithoutDeploy
expr: |
(changes(process_start_time_seconds{job=~"skylight|trastevere-skylight"}[5m]) > 0)
and on (job)
(count by (job) (count_over_time(skylight_build_info{job=~"skylight|trastevere-skylight"}[5m])) == 1)
labels:
severity: ticket
annotations:
summary: '{{ $labels.job }} restarted without a new build'
- alert: BuildVersionSkew
expr: count(count by (version) (sunlight_build_info{job=~"tuscolo|trastevere|navigli|loreto|golf"})) > 1
for: 7d
labels:
severity: ticket
annotations:
summary: '{{ $value }} different sunlight versions have been deployed for a week'
- alert: SunlightTorExitListStale
expr: |
(sum by (job) (increase(sunlight_tor_exit_fetches_total{job=~"tuscolo|trastevere|navigli|loreto", outcome!~"success|unchanged"}[3h])) > 0)
unless on (job)
(sum by (job) (increase(sunlight_tor_exit_fetches_total{job=~"tuscolo|trastevere|navigli|loreto", outcome=~"success|unchanged"}[3h])) > 0)
unless on (job)
(sum by (job) (sunlight_tor_exit_fetches_total{job=~"tuscolo|trastevere|navigli|loreto", outcome=~"success|unchanged"}) > 0
and on (job) time() - process_start_time_seconds{job=~"tuscolo|trastevere|navigli|loreto"} < 3 * 3600)
labels:
severity: ticket
annotations:
summary: '{{ $labels.job }} has not fetched the Tor exit list successfully in 3h'
description: Tor exits are rate-limited per address until a fetch succeeds. The sunlight journal has the fetch error.
- name: sunlight-log-staging
rules:
- alert: StagingCheckpointStale
expr: |
(time() - sunlight_tree_timestamp_seconds{job=~"navigli|loreto"} > 30)
and on (job, log)
(sunlight_config_notafter_end_timestamp_seconds{job=~"navigli|loreto"} > time())
for: 1m
labels:
severity: ticket
annotations:
summary: 'staging {{ $labels.log }} checkpoint is {{ $value | humanizeDuration }} old'
- alert: StagingSequencingFailed
expr: increase(sunlight_sequencing_rounds_total{job=~"navigli|loreto", error!=""}[10m]) > 0
labels:
severity: ticket
annotations:
summary: 'staging {{ $labels.log }} had {{ $value | humanize }} failed sequencing rounds in 10m ({{ $labels.error }})'
- alert: StagingSubmissionSaturated
expr: |
(job_log:sunlight_admission_utilization:rate15m{job=~"navigli|loreto"} > 0.4)
and on (job, log)
(job_log:sunlight_cache_get:rate15m > 10)
for: 5m
labels:
severity: ticket
annotations:
summary: 'staging {{ $labels.log }} admission path is {{ $value | humanizePercentage }} busy'
- alert: StagingOrganicShedding
expr: job_log:sunlight_addchain_organic_shed:rate1m{job=~"navigli|loreto"} > 0.5
for: 5m
labels:
severity: ticket
annotations:
summary: 'staging {{ $labels.log }} is rejecting {{ $value | humanize }}/s of non-low-priority submissions'
- alert: StagingTargetDown
expr: up{job=~"navigli|loreto"} == 0
for: 5m
labels:
severity: ticket
annotations:
summary: '{{ $labels.job }} ({{ $labels.instance }}) is down or unreachable'
- alert: StagingProcessNotRestartingDaily
expr: time() - process_start_time_seconds{job=~"navigli|loreto"} > 26 * 3600
labels:
severity: ticket
annotations:
summary: '{{ $labels.job }} has been up {{ $value | humanizeDuration }}; RuntimeMaxSec=1d is not restarting it'
- name: sunlight-dedup-cache
rules:
- alert: SunlightDedupCacheDegraded
expr: |
(job_log:sunlight_cache_get_seconds:avg15m{job=~"tuscolo|trastevere"} > 0.0008)
and on (job, log)
(job_log:sunlight_cache_get:rate15m > 10)
for: 30m
labels:
severity: ticket
annotations:
summary: '{{ $labels.log }} dedup cache lookups average {{ $value | humanizeDuration }} (healthy 0.2-0.5ms)'
description: The dedup cache no longer fits in the ARC. Check what is using host memory.
- alert: SunlightCachePutCostHigh
expr: |
(job_log:sunlight_cache_put_cost_seconds:avg15m{job=~"tuscolo|trastevere"} > 0.0015)
and on (job, log)
(job_log:sunlight_cache_put_entries:rate15m > 5)
for: 30m
labels:
severity: ticket
annotations:
summary: '{{ $labels.log }} dedup cache inserts cost {{ $value | humanizeDuration }} per entry'
- alert: SunlightCacheErrors
expr: |
sum by (job, log) (increase(sunlight_cache_put_errors_total{job=~"tuscolo|trastevere|navigli|loreto"}[1h])) > 0
or
sum by (job, log) (increase(sunlight_cache_checkpoint_errors_total{job=~"tuscolo|trastevere|navigli|loreto"}[1h])) > 0
or
sum by (job, log) (increase(sunlight_staging_discard_errors_total{job=~"tuscolo|trastevere|navigli|loreto"}[1h])) > 0
labels:
severity: ticket
annotations:
summary: '{{ $labels.log }} had {{ $value | humanize }} dedup cache or staging errors in 1h'
- name: skylight
rules:
- alert: SkylightHealthFailing
expr: sum by (job) (increase(skylight_http_requests_total{job=~"skylight|trastevere-skylight", kind="health", code!="200"}[5m])) > 2
labels:
severity: page
annotations:
summary: '{{ $labels.job }} /health failed {{ $value | humanize }} times in 5m'
description: The /health response body names the failing log or witness and the reason.
- alert: SkylightHealthUnprobed
expr: sum by (job) (increase(skylight_http_requests_total{job=~"skylight|trastevere-skylight", kind="health"}[10m])) == 0
for: 30m
labels:
severity: ticket
annotations:
summary: '{{ $labels.job }} /health has not been probed for 30m'
- alert: SkylightServerErrors
expr: sum by (job) (rate(skylight_http_requests_total{job=~"skylight|trastevere-skylight", code=~"5..", kind!="health"}[5m])) > 0.1
for: 10m
labels:
severity: ticket
annotations:
summary: '{{ $labels.job }} is serving {{ $value | humanize }}/s HTTP 5xx'
- name: witness-golf
rules:
- alert: WitnessGolfSLOFastBurn
expr: sum_over_time(golf:witness_slo_bad_minute[10m]) >= 1
labels:
severity: ticket
annotations:
summary: 'witness golf served a 5xx or a >1s add-checkpoint in the last 10 minutes'
- alert: WitnessGolfSLOSlowBurn
expr: |
(avg_over_time(golf:witness_slo_bad_minute[6h]) > 0.06 and avg_over_time(golf:witness_slo_bad_minute[30m]) > 0.06)
or
(avg_over_time(golf:witness_slo_bad_minute[3d]) > 0.01 and avg_over_time(golf:witness_slo_bad_minute[6h]) > 0.01)
labels:
severity: ticket
annotations:
summary: 'witness golf is over its 1% budget of bad minutes ({{ $value | humanizePercentage }})'
- alert: WitnessGolfProbeSilent
expr: increase(sunlight_witness_add_checkpoint_requests_total{job="golf", origin="geomys.org/witness/test-log", error=""}[15m]) == 0
labels:
severity: ticket
annotations:
summary: 'witness golf has not served the uptime.geomys.org probe for 15 minutes'
- alert: WitnessGolfLoadHigh
expr: sum(rate(sunlight_witness_http_requests_total{job="golf"}[5m])) > 5
for: 10m
labels:
severity: ticket
annotations:
summary: 'witness golf is handling {{ $value | humanize }} requests/s'
- alert: WitnessGolfLogListStale
expr: time() - sunlight_witness_log_list_pull_timestamp_seconds{job="golf"} > 7200
for: 10m
labels:
severity: ticket
annotations:
summary: 'witness golf has not pulled {{ $labels.list }} for {{ $value | humanizeDuration }}'
- name: witness-staging
rules:
- alert: StagingWitnessLogListStale
expr: time() - sunlight_witness_log_list_pull_timestamp_seconds{job="navigli"} > 7200
for: 10m
labels:
severity: ticket
annotations:
summary: 'staging witness has not pulled {{ $labels.list }} for {{ $value | humanizeDuration }}'
- name: host-zfs
rules:
- alert: ZFSArcThrottle
expr: instance:node_zfs_arc_overflow_bytes > 2 * 1024 * 1024 * 1024
for: 15m
labels:
severity: ticket
annotations:
summary: '{{ $labels.instance }} ZFS ARC is {{ $value | humanize1024 }}B over its target'
description: >-
ARC allocations are throttled, which slows dedup cache lookups. Usually pinned metadata
(see ZFSArcPinned); system.slice memory.reclaim or restarting sunlight-staging relieves it.
- alert: ZFSArcPruneStorm
expr: rate(node_zfs_arc_arc_prune{job=~"node|trastevere-node"}[5m]) > 1000
for: 15m
labels:
severity: ticket
annotations:
summary: '{{ $labels.instance }} ZFS arc_prune is running {{ $value | humanize }}/s'
- alert: ZFSArcPinned
expr: instance:node_zfs_arc_pinned_ratio > 0.6
for: 6h
labels:
severity: ticket
annotations:
summary: '{{ $labels.instance }} {{ $value | humanizePercentage }} of the ZFS ARC target is unevictable metadata'
description: Something holds millions of inodes. Find the cgroup with the reclaimable slab.
- alert: ZFSDnodeOverLimit
expr: node_zfs_arc_dnode_size{job=~"node|trastevere-node"} / node_zfs_arc_arc_dnode_limit{job=~"node|trastevere-node"} > 0.9
for: 1h
labels:
severity: ticket
annotations:
summary: '{{ $labels.instance }} ZFS dnode cache is at {{ $value | humanizePercentage }} of arc_dnode_limit'
- alert: ZFSTxgSlow
expr: |
zfs_txg_max_sync_seconds{job=~"node|trastevere-node"} > 2
unless on(instance, pool) max_over_time(zfs_pool_scan_active[15m]) == 1
for: 10m
labels:
severity: ticket
annotations:
summary: '{{ $labels.instance }} pool {{ $labels.pool }} had a {{ $value | humanizeDuration }} txg sync'
- alert: ZFSPoolUnhealthy
expr: node_zfs_zpool_state{job=~"node|trastevere-node", state!="online"} == 1
for: 5m
labels:
severity: ticket
annotations:
summary: '{{ $labels.instance }} pool {{ $labels.zpool }} is {{ $labels.state }}'
description: Check zpool status -x.
- alert: ZFSDeviceErrors
expr: zfs_vdev_errors{job=~"node|trastevere-node"} > 0
labels:
severity: ticket
annotations:
summary: '{{ $labels.instance }} {{ $labels.pool }}/{{ $labels.vdev }} has {{ $value }} {{ $labels.type }} errors'
description: Check zpool status -v, then zpool clear once resolved.
- alert: ZFSDataErrors
expr: zfs_pool_data_errors{job=~"node|trastevere-node"} > 0
labels:
severity: ticket
annotations:
summary: '{{ $labels.instance }} pool {{ $labels.pool }} has {{ $value }} permanent data errors'
description: zpool status -v lists the affected files.
- alert: MDArrayDegraded
expr: |
node_md_disks{job=~"node|trastevere-node", state="failed"} > 0
or
node_md_disks{job=~"node|trastevere-node", state="active"} < ignoring(state) node_md_disks_required
for: 5m
labels:
severity: ticket
annotations:
summary: '{{ $labels.instance }} RAID array {{ $labels.device }} is degraded'
- name: host
rules:
- alert: HostPositiveDentriesHigh
expr: instance:node_dentry_positive > 3e6
for: 30m
labels:
severity: ticket
annotations:
summary: '{{ $labels.instance }} has {{ $value | humanize }} positive dentries'
description: Each one pins a ZFS znode. Find the cgroup with the reclaimable slab (cgroup_memory_stat_bytes, system.slice minus its children for exited units).
- alert: HostOOMKill
expr: increase(node_vmstat_oom_kill{job=~"node|trastevere-node"}[10m]) > 0
labels:
severity: ticket
annotations:
summary: '{{ $labels.instance }} had {{ $value | humanize }} OOM kill(s)'
- alert: ProcessMemoryLeak
expr: |
(min_over_time(deriv(process_resident_memory_bytes{job=~"tuscolo|trastevere|navigli|loreto|golf|skylight|trastevere-skylight"}[6h])[1d:5m]) > 1024 * 1024 / 3600)
and
(time() - process_start_time_seconds{job=~"tuscolo|trastevere|navigli|loreto|golf|skylight|trastevere-skylight"} > 10 * 86400)
for: 1d
labels:
severity: ticket
annotations:
summary: '{{ $labels.job }} memory has grown continuously for 2 days'
- alert: ProcessMemoryHigh
expr: process_resident_memory_bytes{job=~"tuscolo|trastevere|navigli|loreto|golf|skylight|trastevere-skylight"} > 6 * 1024 * 1024 * 1024
for: 30m
labels:
severity: ticket
annotations:
summary: '{{ $labels.job }} is using {{ $value | humanize1024 }}B of memory'
- alert: HostTCPTimeouts
expr: |
count_over_time((
rate(node_netstat_TcpExt_TCPTimeouts{job=~"node|trastevere-node"}[1m])
- rate(node_netstat_TcpExt_TCPSynRetrans{job=~"node|trastevere-node"}[1m])
> 120)[15m:1m]) >= 8
labels:
severity: ticket
annotations:
summary: '{{ $labels.instance }} had over 120/s TCP retransmission timeouts in {{ $value }} of the last 15 minutes (likely packet loss)'
- alert: HostLinkSaturated
expr: |
rate(node_network_transmit_bytes_total{job="trastevere-node", device="ens2f1np1"}[5m]) * 8 > 900e6
or
rate(node_network_receive_bytes_total{job="trastevere-node", device="ens2f1np1"}[5m]) * 8 > 900e6
for: 5m
labels:
severity: ticket
annotations:
summary: '{{ $labels.instance }} {{ $labels.device }} is moving {{ $value | humanize }}bps on a 1 Gbps link'
- alert: ZFSPoolFilling
expr: |
predict_linear(zfs_dataset_available_bytes{job=~"node|trastevere-node", dataset="tank/logs"}[7d], 60 * 86400) < 0
or
zfs_dataset_available_bytes{job=~"node|trastevere-node", dataset="tank/logs"} < 1e12
for: 6h
labels:
severity: ticket
annotations:
summary: '{{ $labels.instance }} tank is projected to fill within 60 days'
- alert: RootFilesystemLow
expr: node_filesystem_avail_bytes{job=~"node|trastevere-node", mountpoint="/"} / node_filesystem_size_bytes{job=~"node|trastevere-node", mountpoint="/"} < 0.10
for: 30m
labels:
severity: ticket
annotations:
summary: '{{ $labels.instance }} root filesystem has {{ $value | humanizePercentage }} free'
- alert: DiskHealthWarning
expr: |
nvme_critical_warning{job=~"node|trastevere-node"} > 0
or
smartmon_device_smart_healthy{job=~"node|trastevere-node"} == 0
for: 10m
labels:
severity: ticket
annotations:
summary: '{{ $labels.instance }} reports a disk health warning ({{ $labels.device }})'
- alert: HostNodeExporterDown
expr: up{job=~"node|trastevere-node"} == 0
for: 5m
labels:
severity: ticket
annotations:
summary: '{{ $labels.job }} ({{ $labels.instance }}) node exporter is down or unreachable'
- name: mechanism-health
rules:
- alert: SystemdUnitFailed
expr: |
node_systemd_unit_state{job=~"node|trastevere-node", state="failed",
name=~"(sunlight|sunlight-staging|sunlight-golf|skylight|caddy|public-config|prometheus|prometheus-node-exporter|zfs-textfile|zfs-zed|zfs-mount|zfs-import-cache|partial-aftersun|partial-aftersun-staging|memcg-reclaim@skylight|heliograph-dashboard|heliograph-dashboard-trastevere|networking)\\.service"} == 1
for: 15m
labels:
severity: ticket
annotations:
summary: '{{ $labels.instance }} unit {{ $labels.name }} has failed'
- alert: ReclaimTimerNotRunning
expr: |
(time() - node_systemd_timer_last_trigger_seconds{job=~"node|trastevere-node", name="memcg-reclaim@skylight.timer"} > 7200)
and
(node_systemd_timer_last_trigger_seconds{job=~"node|trastevere-node", name="memcg-reclaim@skylight.timer"} > 0)
for: 10m
labels:
severity: ticket
annotations:
summary: '{{ $labels.instance }} memcg-reclaim@skylight.timer last fired {{ $value | humanizeDuration }} ago'
- alert: AftersunNotRunning
expr: time() - node_textfile_mtime_seconds{job=~"node|trastevere-node", file=~".*/partial-aftersun(-staging)?\\.prom"} > 1800
for: 5m
labels:
severity: ticket
annotations:
summary: '{{ $labels.instance }} partial-aftersun has not completed a run for {{ $value | humanizeDuration }}'
- alert: AftersunUnhealthy
expr: |
partial_aftersun_success{job=~"node|trastevere-node"} == 0
or
partial_aftersun_duration_seconds{job=~"node|trastevere-node"} > 30
for: 15m
labels:
severity: ticket
annotations:
summary: '{{ $labels.instance }} partial-aftersun for {{ $labels.log }} is failing or slow'
- alert: TextfileCollectorStale
expr: |
time() - node_textfile_mtime_seconds{job=~"node|trastevere-node", file=~".*/zfs\\.prom"} > 600
or
node_textfile_scrape_error{job=~"node|trastevere-node"} == 1
for: 5m
labels:
severity: ticket
annotations:
summary: '{{ $labels.instance }} textfile metrics are stale or unparsable'
- name: external-monitors
rules:
- alert: ExternalProbeFailures
expr: ct_uptime{job="ctuptime", log=~"https://(tuscolo|trastevere)[0-9]+h[12]\\.sunlight\\.geomys\\.org/"} < 100
labels:
severity: ticket
annotations:
summary: 'ctuptime sees {{ $labels.log }} {{ $labels.endpoint }} at {{ $value }}% uptime'
- alert: ExternalFeedStale
expr: |
time() - ct_uptime_timestamp{job="ctuptime"} > 3 * 86400
or
time() - sct_auditing_scrape_timestamp_seconds{job="sctdelay"} > 12 * 3600
or
up{job=~"ctuptime|sctdelay|sslmate"} == 0
for: 2h
labels:
severity: ticket
annotations:
summary: 'external monitor feed {{ $labels.job }} is stale or unreachable'
- alert: SSLMateInconsistentSTH
expr: sslmate_ctlog_inconsistent_sths{job="sslmate", log_name=~"Geomys (Tuscolo|Trastevere) .*"} > 0
labels:
severity: page
annotations:
summary: 'SSLMate reports {{ $value }} inconsistent STHs for {{ $labels.log_name }}'
description: SSLMate saw two tree heads for this log that can't both be valid.
- alert: StagingSSLMateInconsistentSTH
expr: sslmate_ctlog_inconsistent_sths{job="sslmate", log_name=~"Geomys (Navigli|Loreto) .*"} > 0
labels:
severity: ticket
annotations:
summary: 'SSLMate reports {{ $value }} inconsistent STHs for {{ $labels.log_name }}'
- alert: SSLMateSTHErrors
expr: sslmate_ctlog_get_sth_1d{job="sslmate", log_name=~"Geomys .*", result="error"} >= 3
labels:
severity: ticket
annotations:
summary: 'SSLMate failed {{ $value }} checkpoint fetches from {{ $labels.log_name }} in the last day'
- alert: SSLMateSTHStale
expr: time() - sslmate_ctlog_max_sth_timestamp{job="sslmate", log_name=~"Geomys .*", log_status="active"} / 1000 > 3600
for: 10m
labels:
severity: ticket
annotations:
summary: 'SSLMate has not seen a fresh checkpoint from {{ $labels.log_name }} for {{ $value | humanizeDuration }}'
- alert: ChromeSCTIngestionDelayed
expr: sct_auditing_ingestion_delay_seconds{job="sctdelay", log_name=~"Geomys .*", active="true"} > 86400
for: 1h
labels:
severity: ticket
annotations:
summary: 'Chrome SCT auditing has not ingested from {{ $labels.log_name }} for {{ $value | humanizeDuration }}'
- name: meta
rules:
- alert: PrometheusUnhealthy
expr: |
increase(prometheus_rule_evaluation_failures_total[15m]) > 0
or increase(prometheus_rule_group_iterations_missed_total[15m]) > 0
or prometheus_config_last_reload_successful == 0
or increase(prometheus_tsdb_compactions_failed_total[1h]) > 0
or increase(prometheus_tsdb_wal_corruptions_total[1h]) > 0
or count_over_time((increase(prometheus_notifications_dropped_total[90s]) > 0)[15m:1m]) > 2
for: 5m
labels:
severity: ticket
annotations:
summary: 'Prometheus is failing to evaluate rules, reload, compact, or deliver notifications'
- alert: StagingPageLeak
expr: |
ALERTS{alertstate="firing", severity="page", job=~"navigli|loreto"}
or ALERTS{alertstate="firing", severity="page", log=~"(navigli|loreto).*"}
or ALERTS{alertstate="firing", severity="page", log_name=~"Geomys (Navigli|Loreto).*"}
labels:
severity: ticket
annotations:
summary: 'page alert {{ $labels.alertname }} is firing for staging'