================================================
FILE: clan-service-modules/machine-type/default.nix
================================================
{
_class = "clan.service";
manifest.name = "machine-type";
manifest.readme = "Machine classification/profiles";
roles.server.perInstance.nixosModule = ./server.nix;
roles.server.description = "Server machine settings, no GUI";
roles.desktop.perInstance.nixosModule = ./desktop.nix;
roles.desktop.description = "Desktop machine settings, including wayland and sway";
roles.mobile.perInstance.nixosModule = ./mobile.nix;
roles.mobile.description = "Mobile/ARM device settings, lightweight desktop";
# Common configuration for all macine types
perMachine.nixosModule =
{ lib, ... }:
{
security.acme.acceptTerms = true;
security.acme.defaults.email = lib.mkDefault "letsencrypt@pablo.tools";
clan.core.settings.state-version.enable = true;
hardware.enableRedistributableFirmware = true;
hardware.enableAllHardware = true;
};
}
================================================
FILE: clan-service-modules/machine-type/desktop.nix
================================================
{
config,
pkgs,
lib,
nur,
flake-self,
wl-harmonograph,
promterm,
home-manager,
...
}:
{
imports = [
./nextcloud-desktop.nix
home-manager.nixosModules.home-manager
];
services.fwupd.enable = true;
services.acpid.enable = true;
services.upower.enable = true;
# To build raspi images
boot.binfmt.emulatedSystems = [ "aarch64-linux" ];
# Enable networkmanager
networking.networkmanager.enable = true;
# Often hangs
systemd.services = {
NetworkManager-wait-online.enable = lib.mkForce false;
systemd-networkd-wait-online.enable = lib.mkForce false;
};
hardware.keyboard.qmk.enable = true;
services.udev.packages = [
# pkgs.via
pkgs.qmk-udev-rules # For QMK/Via
pkgs.libsigrok # For pulseview
];
# DON'T set useGlobalPackages! It's not necessary in newer
# home-manager versions and does not work with configs using
# nixpkgs.config`
home-manager.useUserPackages = true;
# Backup files before overwriting them with home-manager
home-manager.backupFileExtension = "hm-backup";
# Pass all flake inputs to home-manager modules aswell so we can use them
# there.
# home-manager.extraSpecialArgs = flake-self.inputs;
home-manager.extraSpecialArgs = {
inherit
wl-harmonograph
flake-self
nur
promterm
;
# Pass system configuration (top-level "config") to home-manager modules,
# so we can access it's values for conditional statements. Writing is NOT possible!
system-config = config;
};
nixpkgs.overlays = [
nur.overlays.default
flake-self.overlays.default
];
# TODO parametrize the username
home-manager.users.pinpox = flake-self.homeConfigurations.desktop;
# Hardware accelleration
hardware.graphics = {
enable = true;
enable32Bit = true;
};
pinpox = {
defaults = {
bluetooth.enable = true;
environment.enable = true;
storagebox = {
enable = true;
mountOnAccess = true;
};
fonts.enable = true;
locale.enable = true;
locale.automatic-timezone = true;
networking.enable = true;
nix.enable = true;
sound.enable = true;
zsh.enable = true;
yubikey.enable = true;
lvm-grub.enable = true;
# home-manager.enable = true;
# home-manager.configuration = flake-self.homeConfigurations.desktop;
};
virtualisation = {
docker.enable =false;
virt-manager.enable = true;
virtualbox.enable = false;
};
services = {
unbound-desktop.enable = false;
wayland.enable = true;
openssh.enable = true;
restic-client = {
enable = true;
backup-paths-onsite = [
"/home/pinpox/Notes"
"/home/pinpox"
"/home/pinpox/.mozilla/firefox/pinpox/places.sqlite"
# "*/.local/share/password-store"
# "*/.passage"
# "*/.gnupg"
# "*/.ssh"
];
backup-paths-offsite = [
"/home/pinpox/.mozilla/firefox/pinpox/places.sqlite"
"/home/pinpox/Notes"
"/home/pinpox"
];
};
};
};
# List packages installed in system profile. To search, run:
# $ nix search wget
environment.systemPackages = with pkgs; [
# irc-announce irc.hackint.org 6697 testbot992 '#lounge-rocks2' 1 "test2"
# pkgs.nur.repos.mic92.irc-announce
firefox
openspec
gcc
comma
acpi
arandr
binutils
file
git
gnumake
killall
lm_sensors
neovim
nixpkgs-review
nix-init
nix-update
nodejs
ripgrep
time
usbutils
wget
];
services.logind.settings.Login.RuntimeDirectorySize = "20G";
boot = {
# Use GRUB2 as EFI boot loader.
loader.grub.useOSProber = true;
tmp.useTmpfs = false;
};
programs.dconf.enable = true;
# For user-space mounting things like smb:// and ssh:// in thunar etc. Dbus
# is required.
services.gvfs = {
enable = true;
# Default package does not support all protocols. Use the full-featured
# gnome version
package = lib.mkForce pkgs.gnome.gvfs;
};
}
================================================
FILE: clan-service-modules/machine-type/mobile.nix
================================================
{
config,
pkgs,
lib,
nur,
flake-self,
promterm,
home-manager,
...
}:
{
imports = [
home-manager.nixosModules.home-manager
];
services.acpid.enable = true;
services.upower.enable = true;
# US QWERTY for TTY (mobile devices don't have colemak keyboards)
console.keyMap = lib.mkForce "us";
# Enable networkmanager
networking.networkmanager.enable = true;
# Often hangs
systemd.services = {
NetworkManager-wait-online.enable = lib.mkForce false;
systemd-networkd-wait-online.enable = lib.mkForce false;
};
# DON'T set useGlobalPackages! It's not necessary in newer
# home-manager versions and does not work with configs using
# nixpkgs.config`
home-manager.useUserPackages = true;
# Backup files before overwriting them with home-manager
home-manager.backupFileExtension = "hm-backup";
# Pass all flake inputs to home-manager modules aswell so we can use them
# there.
home-manager.extraSpecialArgs = {
inherit
flake-self
nur
promterm
;
# Pass system configuration (top-level "config") to home-manager modules,
# so we can access it's values for conditional statements. Writing is NOT possible!
system-config = config;
};
nixpkgs.overlays = [
nur.overlays.default
flake-self.overlays.default
];
# TODO parametrize the username
home-manager.users.pinpox = flake-self.homeConfigurations.mobile;
# Hardware accelleration
hardware.graphics = {
enable = true;
};
pinpox = {
defaults = {
bluetooth.enable = true;
environment.enable = true;
fonts.enable = false; # Disabled - noto-fonts-color-emoji can't cross-compile
locale.enable = true;
locale.automatic-timezone = true;
networking.enable = true;
nix.enable = true;
sound.enable = false;
zsh.enable = true;
};
services = {
wayland.enable = true;
openssh.enable = true;
};
};
# List packages installed in system profile
environment.systemPackages = with pkgs; [
gcc
comma
acpi
binutils
file
git
gnumake
killall
neovim
ripgrep
time
usbutils
wget
];
programs.dconf.enable = true;
# Disable speech-dispatcher to avoid pulling in mbrola-voices (675MB)
services.speechd.enable = lib.mkForce false;
# Override fontconfig defaults to avoid noto-fonts-color-emoji
# which requires Python tools that can't cross-compile
fonts.fontconfig.defaultFonts.emoji = lib.mkForce [ ];
# Disable default font packages (includes noto-fonts-color-emoji)
fonts.enableDefaultPackages = false;
# Disable modemmanager - cross-compilation is broken
networking.modemmanager.enable = false;
}
================================================
FILE: clan-service-modules/machine-type/nextcloud-desktop.nix
================================================
{
# Nextcloud on the desktop
services.davfs2 = {
enable = true;
# settings.globalSection.use_locks = false;
# TODO: Note: Ordinary users can mount a davfs2 file system if they are a
# member of the group dav_group as defined in the system wide
# configuration. Make sure the option 'dav_group' is enabled in the system
# wide configuration file.
};
# fileSystems."/home/pinpox/Nextcloud" = {
# device = "https://files.pablo.tools/remote.php/dav/files/pinpox";
# fsType = "davfs";
# options = [
# "user"
# "rw"
# "noauto" # I'll mount it manually
# ];
# };
}
================================================
FILE: clan-service-modules/machine-type/scanners.nix
================================================
{
hardware.sane.enable = true;
users.users.pinpox.extraGroups = [
"scanner"
"lp"
];
}
================================================
FILE: clan-service-modules/machine-type/server.nix
================================================
{
restic-exporter,
lib,
pkgs,
config,
...
}:
with lib;
{
imports = [
restic-exporter.nixosModules.default
];
# Limit log size for journal
services.journald.extraConfig = "SystemMaxUse=1G";
environment.systemPackages = with pkgs; [
universal-ctags
git
gnumake
go
htop
neovim
nix-index
nixfmt
ripgrep
wget
ncdu
duf
tmux
];
pinpox.defaults = {
environment.enable = true;
locale.enable = true;
nix.enable = true;
zsh.enable = true;
networking.enable = true;
};
pinpox.services.openssh.enable = true;
# Backups
pinpox.services = {
restic-client = {
enable = true;
backup-paths-onsite = [
config.services.postgresqlBackup.location
"/home"
"/root"
];
};
};
# Backup Postgres, if it is running
services.postgresqlBackup = {
enable = config.services.postgresql.enable;
startAt = "*-*-* 01:15:00";
location = "/var/backup/postgresql";
backupAll = true;
};
# Remove wayland dependencies on server machine type
nixpkgs.overlays = [
(final: super: {
passage = super.passage.override {
wl-clipboard = null;
xclip = null;
};
neovim = super.neovim.override {
waylandSupport = false;
};
})
];
}
================================================
FILE: clan-service-modules/monitoring/alert-rules.nix
================================================
{ lib, meta }:
let
# docker's filesystems disappear quickly, leading to false positives
deviceFilter = ''path!~"^(/var/lib/docker|/nix/store).*"'';
in
lib.mapAttrsToList
(name: opts: {
alert = name;
expr = opts.condition;
for = opts.time or "2m";
labels = { };
annotations.description = opts.description;
})
({
# prometheus_too_many_restarts = {
# condition = ''changes(process_start_time_seconds{job=~"prometheus|alertmanager"}[15m]) > 2'';
# description = "Prometheus has restarted more than twice in the last 15 minutes. It might be crashlooping.";
# };
# alert_manager_config_not_synced = {
# condition = ''count(count_values("config_hash", alertmanager_config_hash)) > 1'';
# description = "Configurations of AlertManager cluster instances are out of sync.";
# };
#alert_manager_e2e_dead_man_switch = {
# condition = "vector(1)";
# description = "Prometheus DeadManSwitch is an always-firing alert. It's used as an end-to-end test of Prometheus through the Alertmanager.";
#};
# prometheus_not_connected_to_alertmanager = {
# condition = "prometheus_notifications_alertmanagers_discovered < 1";
# description = "Prometheus cannot connect the alertmanager\n VALUE = {{ $value }}\n LABELS = {{ $labels }}";
# };
# prometheus_rule_evaluation_failures = {
# condition = "increase(prometheus_rule_evaluation_failures_total[3m]) > 0";
# description = "Prometheus encountered {{ $value }} rule evaluation failures, leading to potentially ignored alerts.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}";
# };
# prometheus_template_expansion_failures = {
# condition = "increase(prometheus_template_text_expansion_failures_total[3m]) > 0";
# time = "0m";
# description = "Prometheus encountered {{ $value }} template text expansion failures\n VALUE = {{ $value }}\n LABELS = {{ $labels }}";
# };
# promtail_file_lagging = {
# condition = ''abs(promtail_file_bytes_total - promtail_read_bytes_total) > 1e6'';
# time = "15m";
# description = ''{{ $labels.instance }} {{ $labels.job }} {{ $labels.path }} has been lagging by more than 1MB for more than 15m.'';
# };
filesystem_full_80percent = {
condition = ''100 - ((node_filesystem_avail_bytes{fstype!="rootfs",mountpoint="/"} * 100) / node_filesystem_size_bytes{fstype!="rootfs",mountpoint="/"}) > 80'';
time = "10m";
description = "{{$labels.instance}} device {{$labels.device}} on {{$labels.mountpoint}} got less than 20% space left on its filesystem.";
};
# filesystem_inodes_full = {
# condition = ''disk_inodes_free / disk_inodes_total < 0.10'';
# time = "10m";
# description = "{{$labels.instance}} device {{$labels.device}} on {{$labels.mountpoint}} got less than 10% inodes left on its filesystem.";
# };
# daily_task_not_run = {
# # give 6 hours grace period
# condition = ''time() - task_last_run{state="ok",frequency="daily"} > (24 + 6) * 60 * 60'';
# description = "{{$labels.instance}}: {{$labels.name}} was not run in the last 24h";
# };
# TODO for backups:
# - Has not run
# - No data for a wheek
# - Too old
# - to drastic file or size change
# description = "status of ${name} is unknown: no data for a day";
# description = "{{$labels.instance}}: {{$labels.name}} was not run in the last week";
# description = "status of restic is unknown: no data for a week";
# homeassistant = {
# condition = ''
# homeassistant_entity_available{domain="persistent_notification", entity!="persistent_notification.http_login"} >= 0'';
# description =
# "homeassistant notification {{$labels.entity}} ({{$labels.friendly_name}}): {{$value}}";
# };
swap_using_20percent = {
condition = "node_memory_SwapTotal_bytes - (node_memory_SwapCached_bytes + node_memory_SwapFree_bytes) > node_memory_SwapTotal_bytes * 0.2";
time = "30m";
description = "{{$labels.instance}} is using 20% of its swap space for at least 30 minutes.";
};
systemd_service_failed = {
condition = ''node_systemd_unit_state{state="failed"} == 1'';
description = "{{$labels.instance}} failed to (re)start service {{$labels.name}}.";
};
restic_backup_too_old = {
condition = ''(time() - restic_snapshots_latest_time)/(60*60) > 24'';
description = "{{$labels.instance}} not backed up for more than 24 hours. ({{$value}})";
};
host_down = {
condition = ''up{job="node-stats", instance!~"limette.${meta.domain}:9100|kartoffel.${meta.domain}:9100"} == 0'';
description = "{{$labels.instance}} is down!";
};
# service_not_running = {
# condition = ''systemd_units_active_code{name=~"teamspeak3-server.service|tt-rss.service", sub!="running"}'';
# description = "{{$labels.instance}} should have a running {{$labels.name}}.";
# };
ram_using_90percent = {
condition = "node_memory_Buffers_bytes + node_memory_MemFree_bytes + node_memory_Cached_bytes < node_memory_MemTotal_bytes * 0.1";
time = "1h";
description = "{{$labels.instance}} is using at least 90% of its RAM for at least 1 hour.";
};
cpu_using_90percent = {
condition = ''100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) >= 90'';
time = "10m";
description = "{{$labels.instance}} is running with cpu usage > 90% for at least 10 minutes: {{$value}}";
};
reboot = {
condition = "node_boot_time_seconds < 300";
description = "{{$labels.instance}} just rebooted.";
};
uptime = {
condition = "(time() - node_boot_time_seconds ) / (60*60*24) > 30";
description = "Uptime monster: {{$labels.instance}} has been up for more than 30 days.";
};
flake_nixpkgs_outdated = {
condition = ''(time() - flake_input_last_modified{input="nixpkgs"}) / (60*60*24) > 30'';
description = "Nixpkgs outdated: Nixpkgs on {{$labels.instance}} has not been updated in 30 days";
};
/*
ping = {
condition = "ping_result_code{type!='mobile'} != 0";
description = "{{$labels.url}}: ping from {{$labels.instance}} has failed!";
};
ping_high_latency = {
condition = "ping_average_response_ms{type!='mobile'} > 5000";
description = "{{$labels.instance}}: ping probe from {{$labels.source}} is encountering high latency!";
};
*/
http_status = {
condition = ''probe_http_status_code{instance!~"https://megaclan3000.de"} != 200'';
description = "http request failed from {{$labels.instance}}: {{$labels.result}}!";
};
/*
http_match_failed = {
condition = "http_response_response_string_match == 0";
description = "{{$labels.server}} : http body not as expected; status code: {{$labels.status_code}}!";
};
dns_query = {
condition = "dns_query_result_code != 0";
description = "{{$labels.domain}} : could retrieve A record {{$labels.instance}} from server {{$labels.server}}: {{$labels.result}}!";
};
secure_dns_query = {
condition = "secure_dns_state != 0";
description = "{{$labels.domain}} : could retrieve A record {{$labels.instance}} from server {{$labels.server}}: {{$labels.result}} for protocol {{$labels.protocol}}!";
};
connection_failed = {
condition = "net_response_result_code != 0";
description = "{{$labels.server}}: connection to {{$labels.port}}({{$labels.protocol}}) failed from {{$labels.instance}}";
};
healthchecks = {
condition = "hc_check_up == 0";
description = "{{$labels.instance}}: healtcheck {{$labels.job}} fails!";
};
*/
cert_expiry = {
condition = "(probe_ssl_earliest_cert_expiry - time())/(3600*24) < 30";
description = "{{$labels.instance}}: The TLS certificate will expire in less than 30 days: {{$value}}s";
};
# ignore devices that disabled S.M.A.R.T (example if attached via USB)
# smart_errors = {
# condition = ''smart_device_health_ok{enabled!="Disabled"} != 1'';
# description =
# "{{$labels.instance}}: S.M.A.R.T reports: {{$labels.device}} ({{$labels.model}}) has errors.";
# };
oom_kills = {
condition = "increase(node_vmstat_oom_kill[5m]) > 0";
description = "{{$labels.instance}}: OOM kill detected";
};
/*
unusual_disk_read_latency = {
condition =
"rate(diskio_read_time[1m]) / rate(diskio_reads[1m]) > 0.1 and rate(diskio_reads[1m]) > 0";
description = ''
{{$labels.instance}}: Disk latency is growing (read operations > 100ms)
'';
};
unusual_disk_write_latency = {
condition =
"rate(diskio_write_time[1m]) / rate(diskio_write[1m]) > 0.1 and rate(diskio_write[1m]) > 0";
description = ''
{{$labels.instance}}: Disk latency is growing (write operations > 100ms)
'';
};
*/
host_memory_under_memory_pressure = {
condition = "rate(node_vmstat_pgmajfault[1m]) > 1000";
description = "{{$labels.instance}}: The node is under heavy memory pressure. High rate of major page faults: {{$value}}";
};
# ext4_errors = {
# condition = "ext4_errors_value > 0";
# description =
# "{{$labels.instance}}: ext4 has reported {{$value}} I/O errors: check /sys/fs/ext4/*/errors_count";
# };
# alerts_silences_changed = {
# condition = ''abs(delta(alertmanager_silences{state="active"}[1h])) >= 1'';
# description =
# "alertmanager: number of active silences has changed: {{$value}}";
# };
})
================================================
FILE: clan-service-modules/monitoring/alertmanager-irc-relay.nix
================================================
{ pkgs, config, ... }:
let
am-irc-conf = {
# listening host/port.
http_host = "localhost";
http_port = 8667;
# Connect to this IRC host/port.
irc_host = "irc.hackint.org";
irc_port = 6697;
# irc_host_password = "myserver_password";
irc_nickname = "alertus-maximus";
irc_nickname_password = "mynickserv_key";
irc_realname = "myrealname";
irc_channels = [ { name = "#lounge-rocks-log"; } ];
msg_once_per_alert_group = false;
# Use PRIVMSG instead of NOTICE (default) to send messages.
use_privmsg = true;
# Define how IRC messages should be formatted.
msg_template = "⚠ ⚠ ⚠ [{{.Labels.instance}}] - {{ .Labels.alertname }} is {{.Status}} ⚠ ⚠ ⚠ {{.Annotations.description}} (@pinpox act accordingly)";
# Note: When sending only one message per alert group the default
# msg_template is set to
# "Alert {{ .GroupLabels.alertname }} for {{ .GroupLabels.job }} is {{ .Status }}"
# Set the internal buffer size for alerts received but not yet sent to IRC.
alert_buffer_size = 2048;
};
confPath = pkgs.writeText "config.yml" (builtins.toJSON am-irc-conf);
in
{
# User and group
users.groups."alertmanager-irc-relay" = { };
users.users."alertmanager-irc-relay" = {
isSystemUser = true;
group = "alertmanager-irc-relay";
};
# Service
systemd.services.alertmanager-irc-relay = {
wantedBy = [ "multi-user.target" ];
serviceConfig = {
ExecStart = "${pkgs.alertmanager-irc-relay}/bin/alertmanager-irc-relay --config ${confPath}";
User = config.users.users.alertmanager-irc-relay.name;
Group = config.users.users.alertmanager-irc-relay.name;
};
};
}
================================================
FILE: clan-service-modules/monitoring/blackbox.nix
================================================
{ pkgs, ... }:
{
services.prometheus.exporters.blackbox = {
enable = true;
# Default port is 9115
# Listen on 0.0.0.0, but we only open the firewall for wg-clan
openFirewall = false;
configFile = pkgs.writeTextFile {
name = "blackbox-exporter-config";
text = ''
modules:
http_2xx:
prober: http
timeout: 5s
http:
valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
valid_status_codes: [] # Defaults to 2xx
method: GET
no_follow_redirects: false
fail_if_ssl: false
fail_if_not_ssl: false
tls_config:
insecure_skip_verify: false
preferred_ip_protocol: "ip4" # defaults to "ip6"
ip_protocol_fallback: true # fallback to "ip6"
'';
};
};
networking.firewall.interfaces.wg-clan.allowedTCPPorts = [ 9115 ];
}
================================================
FILE: clan-service-modules/monitoring/default.nix
================================================
{ lib, ... }:
{
_class = "clan.service";
manifest.name = "monitoring";
manifest.description = "Prometheus/Loki/Grafana monitoring stack";
manifest.readme = ''
Self-hosted monitoring with one role per component (prometheus, loki,
grafana, blackbox, node-exporter, alertmanager-irc-relay)
'';
manifest.categories = [ "Monitoring" ];
manifest.exports.out = [ "endpoints" "auth" ];
roles.node-exporter = {
description = "Prometheus node-exporter for host metrics (assign to every host you want scraped)";
perInstance.nixosModule = ./node-exporter.nix;
};
roles.loki = {
description = "Loki log aggregation server";
perInstance.nixosModule = ./loki.nix;
};
roles.grafana = {
description = "Grafana dashboard server";
interface =
{ lib, meta, ... }:
{
options = {
domain = lib.mkOption {
type = lib.types.str;
default = "status.${meta.domain}";
example = "dashboards.example.com";
description = "Domain for grafana";
};
oidc = {
enable = lib.mkEnableOption "OIDC-only authentication (e.g. via Authelia). Disables the local login form.";
issuer = lib.mkOption {
type = lib.types.str;
default = "";
example = "https://auth.example.com";
description = "Base URL of the OIDC provider (Authelia). Used to derive auth/token/userinfo endpoints.";
};
clientId = lib.mkOption {
type = lib.types.str;
default = "grafana";
description = "OIDC client ID registered with the provider";
};
providerName = lib.mkOption {
type = lib.types.str;
default = "Authelia";
description = "Display name shown on the Grafana login button";
};
};
};
};
perInstance =
{
settings,
roles,
meta,
mkExports,
...
}:
let
# TODO The generator shoudl be independeat of authelia or kanidm. Use a dependant one for authelia to create the hash
# TODO maybe we do not need a generato cofig at all, have a default?
# Generator definition without runtimeInputs (pkgs isn't available
# at inventory-eval time). Each nixosModule that declares this
# generator adds runtimeInputs locally where pkgs IS available.
oidcGenerator = lib.optionalAttrs settings.oidc.enable {
share = true;
files.client_secret = {
owner = "grafana";
group = "authelia-main";
mode = "0440";
};
files.client_secret_hash.owner = "authelia-main";
script = ''
mkdir -p $out
openssl rand -hex 32 > $out/client_secret
authelia crypto hash generate argon2 --password "$(cat $out/client_secret)" \
| sed 's/^Digest: //' > $out/client_secret_hash
'';
};
in
{
exports = mkExports (
{ endpoints.hosts = [ settings.domain ]; }
// lib.optionalAttrs settings.oidc.enable {
auth.client = {
clientId = settings.oidc.clientId;
clientName = "Grafana";
redirectUris = [ "https://${settings.domain}/login/generic_oauth" ];
scopes = [
"openid"
"profile"
"email"
"groups"
];
public = false;
};
auth.varsGenerator = oidcGenerator;
}
);
nixosModule = import ./grafana.nix {
inherit settings roles meta oidcGenerator;
};
};
};
roles.blackbox = {
description = "Prometheus blackbox-exporter for HTTP/TLS probes";
perInstance.nixosModule = ./blackbox.nix;
};
roles.alertmanager-irc-relay = {
description = "Relay alertmanager notifications to an IRC channel";
perInstance.nixosModule = ./alertmanager-irc-relay.nix;
};
roles.prometheus = {
description = "Prometheus server (with bundled alertmanager). Discovers node-exporter and blackbox role members automatically.";
interface =
{ lib, meta, ... }:
{
options = {
domain = lib.mkOption {
type = lib.types.str;
default = "prometheus.${meta.domain}";
example = "metrics.example.com";
description = "Domain for the prometheus web UI (reverse proxied via caddy)";
};
blackboxTargets = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ "https://pablo.tools" ];
example = [ "https://github.com" ];
description = "Targets to monitor with the blackbox-exporter";
};
jsonTargets = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
example = [ "http://birne.pin/restic-ahorn.json" ];
description = "Targets to probe with the json-exporter";
};
webExternalUrl = lib.mkOption {
type = lib.types.str;
default = "https://vpn.prometheus.pablo.tools";
description = "Prometheus web external URL";
};
alertmanagerWebExternalUrl = lib.mkOption {
type = lib.types.str;
default = "https://vpn.alerts.pablo.tools";
description = "Alertmanager web external URL";
};
};
};
perInstance =
{
settings,
roles,
meta,
mkExports,
...
}:
let
# oauth2-proxy OIDC generator definition (without runtimeInputs —
# added by each nixosModule where pkgs is available).
# Produces: client_secret (raw), client_secret_hash (argon2 for
# authelia), envfile (oauth2-proxy env with client+cookie secrets).
prometheusOidcGenerator = {
share = true;
files.client_secret = {
owner = "oauth2_proxy";
group = "authelia-main";
mode = "0440";
};
files.client_secret_hash.owner = "authelia-main";
files.envfile = {
owner = "oauth2_proxy";
mode = "0400";
};
script = ''
mkdir -p $out
CLIENT_SECRET=$(openssl rand -hex 32)
COOKIE_SECRET=$(openssl rand -hex 16)
printf '%s' "$CLIENT_SECRET" > $out/client_secret
authelia crypto hash generate argon2 --password "$CLIENT_SECRET" \
| sed 's/^Digest: //' > $out/client_secret_hash
printf 'OAUTH2_PROXY_CLIENT_SECRET=%s\nOAUTH2_PROXY_COOKIE_SECRET=%s\n' \
"$CLIENT_SECRET" "$COOKIE_SECRET" > $out/envfile
'';
};
in
{
exports = mkExports (
{ endpoints.hosts = [ settings.domain ]; }
// {
auth.client = {
clientId = "prometheus";
clientName = "Prometheus";
redirectUris = [ "https://${settings.domain}/oauth2/callback" ];
scopes = [
"openid"
"profile"
"email"
"groups"
];
public = false;
};
auth.varsGenerator = prometheusOidcGenerator;
}
);
nixosModule = import ./prometheus.nix {
inherit settings roles meta prometheusOidcGenerator;
};
};
};
}
================================================
FILE: clan-service-modules/monitoring/grafana.nix
================================================
{ settings, roles, meta, oidcGenerator }:
{ config, lib, pkgs, ... }:
let
prometheusHosts = builtins.attrNames (roles.prometheus.machines or { });
lokiHosts = builtins.attrNames (roles.loki.machines or { });
oidc = settings.oidc;
in
{
# SMTP password file
clan.core.vars.generators."grafana".prompts.smtp-password.persist = true;
# OIDC client secret — declared here so the producing host (where grafana
# runs) has the files. The same generator definition is also exported via
# auth.varsGenerator so the authelia host declares it too (share=true).
# runtimeInputs is added here (not in the export) because pkgs is only
# available inside nixosModule, not at inventory-eval time.
clan.core.vars.generators."authelia-oidc-${oidc.clientId}" = lib.mkIf oidc.enable (
oidcGenerator
// {
runtimeInputs = with pkgs; [
coreutils
openssl
authelia
gnused
];
}
);
# Grafana secret key for signing
clan.core.vars.generators."grafana-secret-key" = {
files."secret-key".owner = "grafana";
runtimeInputs = [ pkgs.openssl ];
script = ''
openssl rand -hex 32 > "$out/secret-key"
'';
};
# Backup Graphana dir, contains stateful config
pinpox.services.restic-client.backup-paths-offsite = [ "/var/lib/grafana" ];
# Reverse proxy
services.caddy = {
enable = true;
virtualHosts."${settings.domain}".extraConfig = "reverse_proxy 127.0.0.1:9005";
};
# Graphana fronend
services.grafana = {
enable = true;
settings = {
server = {
domain = settings.domain;
root_url = "https://${settings.domain}/";
# Default is 3000
http_port = 9005;
http_addr = "127.0.0.1";
};
security.secret_key = "$__file{${config.clan.core.vars.generators."grafana-secret-key".files."secret-key".path}}";
# Mail notifications
smtp = {
enabled = true;
host = "smtp.sendgrid.net:587";
user = "apikey";
passwordFile = "${config.clan.core.vars.generators."grafana".files."smtp-password".path}";
fromAddress = "status@pablo.tools";
};
}
// lib.optionalAttrs oidc.enable {
# SCIM is enabled by default in recent Grafana versions and intercepts
# OIDC user provisioning, causing "Failed to create user: user not
# found". Disable it so the standard generic_oauth flow can create users.
feature_toggles.enableSCIM = false;
# OIDC-only login: hide the username/password form, do not auto-create
# local users via signup, redirect straight to the OIDC provider.
auth = {
disable_login_form = true;
signout_redirect_url = "${oidc.issuer}/logout";
# Self-healing OIDC user linking: if user_auth has no row for the
# OIDC subject (e.g. fresh DB restore, or Authelia rebuilt and the
# sub UUID changed), Grafana falls back to matching by email and
# auto-creates the user_auth link. Safe here because Authelia is a
# fully trusted IdP we control.
oauth_allow_insecure_email_lookup = true;
};
users = {
# Must be true so the OIDC provisioning path can create new users.
# disable_login_form = true (above) already prevents the local signup
# form from being rendered, so this only enables OIDC-initiated signup.
allow_sign_up = true;
auto_assign_org = true;
auto_assign_org_role = "Admin";
};
"auth.generic_oauth" = {
enabled = true;
name = oidc.providerName;
icon = "signin";
auto_login = true;
client_id = oidc.clientId;
client_secret = "$__file{${config.clan.core.vars.generators."authelia-oidc-${oidc.clientId}".files.client_secret.path}}";
scopes = "openid profile email groups";
empty_scopes = false;
auth_url = "${oidc.issuer}/api/oidc/authorization";
token_url = "${oidc.issuer}/api/oidc/token";
api_url = "${oidc.issuer}/api/oidc/userinfo";
login_attribute_path = "preferred_username";
email_attribute_path = "email";
name_attribute_path = "name";
groups_attribute_path = "groups";
# Authelia already restricts who can reach the client via its
# authorization policy, so anyone who successfully logs in is granted
# Admin in Grafana.
role_attribute_path = "'Admin'";
allow_sign_up = true;
use_pkce = true;
};
};
# TODO provision the dashboards as currently configured
provision.datasources.settings = {
datasources =
lib.optional (prometheusHosts != [ ]) {
name = "Prometheus";
url = "http://${builtins.head prometheusHosts}.${meta.domain}:9090";
type = "prometheus";
isDefault = true;
}
++ lib.optional (lokiHosts != [ ]) {
name = "loki";
url = "http://${builtins.head lokiHosts}.${meta.domain}:3100";
type = "loki";
};
};
};
}
================================================
FILE: clan-service-modules/monitoring/loki.nix
================================================
{ ... }:
let
port-loki = 3100;
in
{
pinpox.services.restic-client.backup-paths-exclude = [ "/var/lib/loki" ];
networking.firewall = {
enable = true;
interfaces.wg-clan.allowedTCPPorts = [ port-loki ];
};
services.loki = {
enable = true;
configuration = {
auth_enabled = false;
server.http_listen_port = port-loki;
ingester = {
lifecycler = {
address = "0.0.0.0";
ring = {
kvstore.store = "inmemory";
replication_factor = 1;
};
final_sleep = "0s";
};
# Any chunk not receiving new logs in this time will be flushed
chunk_idle_period = "1h";
# All chunks will be flushed when they hit this age, default is 1h
max_chunk_age = "1h";
# Loki will attempt to build chunks up to 1.5MB, flushing first if
# chunk_idle_period or max_chunk_age is reached first
chunk_target_size = 1048576;
# Must be greater than index read cache TTL if using an index cache (Default
# index read cache TTL is 5m)
chunk_retain_period = "30s";
};
schema_config.configs = [
{
from = "2020-10-24";
store = "boltdb-shipper";
object_store = "filesystem";
schema = "v13";
index = {
prefix = "index_";
period = "24h";
};
}
];
storage_config = {
boltdb_shipper = {
active_index_directory = "/var/lib/loki/boltdb-shipper-active";
cache_location = "/var/lib/loki/boltdb-shipper-cache";
# Can be increased for faster performance over longer query periods,
# uses more disk space
cache_ttl = "24h";
};
filesystem.directory = "/var/lib/loki/chunks";
};
limits_config = {
reject_old_samples = true;
reject_old_samples_max_age = "168h";
allow_structured_metadata = false;
};
table_manager = {
retention_deletes_enabled = false;
retention_period = "0s";
};
compactor.working_directory = "/var/lib/loki/boltdb-shipper-compactor";
};
};
}
================================================
FILE: clan-service-modules/monitoring/node-exporter.nix
================================================
{ ... }:
{
services.prometheus.exporters.node = {
enable = true;
# Default port is 9100
# Listen on 0.0.0.0, but we only open the firewall for wg-clan
openFirewall = false;
enabledCollectors = [
"cgroups"
"systemd"
];
extraFlags = [ "--collector.textfile.directory=/etc/nix" ];
};
networking.firewall.interfaces.wg-clan.allowedTCPPorts = [ 9100 ];
}
================================================
FILE: clan-service-modules/monitoring/prometheus.nix
================================================
{ settings, roles, meta, prometheusOidcGenerator }:
{
lib,
pkgs,
config,
flake-self,
pinpox-utils,
...
}:
let
nodeExporterHosts = builtins.attrNames (roles."node-exporter".machines or { });
in
{
clan.core.vars.generators."restic-exporter" = pinpox-utils.mkEnvGenerator [
"RESTIC_PASSWORD"
"AWS_ACCESS_KEY_ID"
"AWS_SECRET_ACCESS_KEY"
"RESTIC_REPOSITORY"
];
# OIDC client secret generator — declared here so the producing host has the
# files. The same definition is exported via auth.varsGenerator so the
# authelia host also declares it (share=true). runtimeInputs added here
# because pkgs isn't available at inventory-eval time.
clan.core.vars.generators."authelia-oidc-prometheus" = prometheusOidcGenerator // {
runtimeInputs = with pkgs; [
coreutils
openssl
authelia
gnused
];
};
# oauth2-proxy sits between Caddy and Prometheus and handles the OIDC
# client flow against Authelia. Prometheus itself stays on 127.0.0.1:9090
# (loopback only); the only way to reach it is through oauth2-proxy on
# 127.0.0.1:4180, which Caddy reverse-proxies as ${settings.domain}.
services.oauth2-proxy = {
enable = true;
provider = "oidc";
clientID = "prometheus";
keyFile = config.clan.core.vars.generators."authelia-oidc-prometheus".files.envfile.path;
oidcIssuerUrl = "https://auth.pablo.tools";
redirectURL = "https://${settings.domain}/oauth2/callback";
upstream = [ "http://127.0.0.1:9090" ];
httpAddress = "http://127.0.0.1:4180";
cookie.secure = true;
cookie.refresh = "1h";
email.domains = [ "*" ];
setXauthrequest = true;
reverseProxy = true;
extraConfig = {
skip-provider-button = true;
# Authelia's prometheus client is registered with require_pkce = true,
# so oauth2-proxy must send a PKCE code_challenge on the authorize
# request and the matching verifier on the token exchange.
code-challenge-method = "S256";
# Authelia's authorization_policies already restrict who can reach this
# client; oauth2-proxy doesn't need to enforce its own allowlist.
};
};
# Reverse proxy for the prometheus web UI. The pki clan service auto-issues
# a TLS cert for ${settings.domain} and prepends a `tls` directive to this
# vhost; dm-dns distributes a CNAME so any clan-internal host can resolve
# it. Caddy hands traffic to oauth2-proxy, which performs the OIDC dance
# against Authelia and (if authorized) proxies upstream to Prometheus.
services.caddy = {
enable = true;
virtualHosts."${settings.domain}".extraConfig = "reverse_proxy 127.0.0.1:4180";
};
services.prometheus = {
enable = true;
# Bind to localhost only — defense in depth on top of the host firewall.
# The only way in from outside the host is via Caddy → oauth2-proxy →
# 127.0.0.1:9090.
listenAddress = "127.0.0.1";
# Disable config checks. They will fail because they run sandboxed and
# can't access external files, e.g. the secrets stored in /run/keys
# https://github.com/NixOS/nixpkgs/blob/d89d7af1ba23bd8a5341d00bdd862e8e9a808f56/nixos/modules/services/monitoring/prometheus/default.nix#L1732-L1738
checkConfig = false;
webExternalUrl = settings.webExternalUrl;
extraFlags = [
"--log.level=debug"
"--storage.tsdb.retention.size='6GB'"
];
ruleFiles = [
(pkgs.writeText "prometheus-rules.yml" (
builtins.toJSON {
groups = [
{
name = "alerting-rules";
rules = import ./alert-rules.nix { inherit lib meta; };
}
];
}
))
];
alertmanagers = [ { static_configs = [ { targets = [ "localhost:9093" ]; } ]; } ];
scrapeConfigs = [
{
job_name = "backup-reports";
scrape_interval = "60m";
metrics_path = "/probe";
static_configs = [ { targets = settings.jsonTargets; } ];
relabel_configs = [
{
source_labels = [ "__address__" ];
target_label = "__param_target";
}
{
source_labels = [ "__param_target" ];
target_label = "instance";
}
{
target_label = "__address__";
replacement = "127.0.0.1:7979"; # The blackbox exporter's real hostname:port.
}
];
}
# TODO: move restic-client to a clan service so this can be discovered
# via roles.