From 7df08cd16db43005075ae02102a4c4af2c399ed2 Mon Sep 17 00:00:00 2001 From: jhodgkin Date: Sun, 12 Jul 2026 20:40:10 -0600 Subject: [PATCH] Add SshHostCollector for .180 (omv) and .171 (ripper) Extends monitoring to the two bare-metal boxes Proxmox can't see. Uses a dedicated ed25519 key with a forced authorized_keys command (see docs/ssh-collector-key-setup.md) so a leaked key can only ever run the fixed read-only stats script, never arbitrary commands. CPU is approximated from 1-min load average / core count (a true utilization % would need two /proc/stat samples). Closes #8. Co-Authored-By: Claude Sonnet 5 --- .env.example | 4 + .gitignore | 1 + apps/api/package-lock.json | 106 ++++++++++++++++++++ apps/api/package.json | 2 + apps/api/src/collectors/sshHost.ts | 149 +++++++++++++++++++++++++++++ apps/api/src/config/index.ts | 26 +++++ apps/api/src/index.ts | 2 + config/hosts.yaml | 22 +++++ docker-compose.yml | 2 + docs/ssh-collector-key-setup.md | 35 +++++++ 10 files changed, 349 insertions(+) create mode 100644 apps/api/src/collectors/sshHost.ts create mode 100644 docs/ssh-collector-key-setup.md diff --git a/.env.example b/.env.example index 2c8913f..d89ad82 100644 --- a/.env.example +++ b/.env.example @@ -24,3 +24,7 @@ COOKIE_SECURE=false PORT=3000 POLL_INTERVAL_SECONDS=30 SNAPSHOT_RETENTION_HOURS=24 + +# Dedicated key for SshHostCollector (omv, ripper) — see docs/ssh-collector-key-setup.md. +# In docker-compose this is mounted from ./ssh/monitor_ed25519 (gitignored, not this path). +SSH_PRIVATE_KEY_PATH=./ssh/monitor_ed25519 diff --git a/.gitignore b/.gitignore index 0955635..f3b7389 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,6 @@ data/ .env .env.* !.env.example +ssh/ *.log .DS_Store diff --git a/apps/api/package-lock.json b/apps/api/package-lock.json index e1df7e5..aed052a 100644 --- a/apps/api/package-lock.json +++ b/apps/api/package-lock.json @@ -15,6 +15,7 @@ "better-sqlite3": "^11.9.1", "dotenv": "^16.4.7", "fastify": "^5.2.1", + "ssh2": "^1.16.0", "undici": "^7.3.0", "yaml": "^2.7.0" }, @@ -22,6 +23,7 @@ "@types/bcryptjs": "^2.4.6", "@types/better-sqlite3": "^7.6.12", "@types/node": "^22.13.5", + "@types/ssh2": "^1.15.1", "tsx": "^4.19.3", "typescript": "^5.7.3" } @@ -765,6 +767,33 @@ "undici-types": "~6.21.0" } }, + "node_modules/@types/ssh2": { + "version": "1.15.5", + "resolved": "https://registry.npmjs.org/@types/ssh2/-/ssh2-1.15.5.tgz", + "integrity": "sha512-N1ASjp/nXH3ovBHddRJpli4ozpk6UdDYIX4RJWFa9L1YKnzdhTlVmiGHm4DZnj/jLbqZpes4aeR30EFGQtvhQQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "^18.11.18" + } + }, + "node_modules/@types/ssh2/node_modules/@types/node": { + "version": "18.19.130", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", + "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@types/ssh2/node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true, + "license": "MIT" + }, "node_modules/abstract-logging": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/abstract-logging/-/abstract-logging-2.0.1.tgz", @@ -804,6 +833,15 @@ } } }, + "node_modules/asn1": { + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", + "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": "~2.1.0" + } + }, "node_modules/atomic-sleep": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", @@ -862,6 +900,15 @@ ], "license": "MIT" }, + "node_modules/bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", + "license": "BSD-3-Clause", + "dependencies": { + "tweetnacl": "^0.14.3" + } + }, "node_modules/bcryptjs": { "version": "2.4.3", "resolved": "https://registry.npmjs.org/bcryptjs/-/bcryptjs-2.4.3.tgz", @@ -935,6 +982,15 @@ "ieee754": "^1.1.13" } }, + "node_modules/buildcheck": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.7.tgz", + "integrity": "sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA==", + "optional": true, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", @@ -966,6 +1022,20 @@ "url": "https://opencollective.com/express" } }, + "node_modules/cpu-features": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.10.tgz", + "integrity": "sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "buildcheck": "~0.0.6", + "nan": "^2.19.0" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -1560,6 +1630,13 @@ "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", "license": "MIT" }, + "node_modules/nan": { + "version": "2.28.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.28.0.tgz", + "integrity": "sha512-fTsDz99OTq2sVePhGdp4qQhggZFtKr64ZNVyVajRKtMOkJxYekplBh577PiJB12v/D3s2E5cGtOI45LWp6rnLQ==", + "license": "MIT", + "optional": true + }, "node_modules/napi-build-utils": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", @@ -1846,6 +1923,12 @@ "node": ">=10" } }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, "node_modules/secure-json-parse": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-4.1.0.tgz", @@ -1982,6 +2065,23 @@ "node": ">= 10.x" } }, + "node_modules/ssh2": { + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.17.0.tgz", + "integrity": "sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==", + "hasInstallScript": true, + "dependencies": { + "asn1": "^0.2.6", + "bcrypt-pbkdf": "^1.0.2" + }, + "engines": { + "node": ">=10.16.0" + }, + "optionalDependencies": { + "cpu-features": "~0.0.10", + "nan": "^2.23.0" + } + }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", @@ -2104,6 +2204,12 @@ "node": "*" } }, + "node_modules/tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==", + "license": "Unlicense" + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", diff --git a/apps/api/package.json b/apps/api/package.json index 08b3d90..629ac18 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -17,6 +17,7 @@ "better-sqlite3": "^11.9.1", "dotenv": "^16.4.7", "fastify": "^5.2.1", + "ssh2": "^1.16.0", "undici": "^7.3.0", "yaml": "^2.7.0" }, @@ -24,6 +25,7 @@ "@types/bcryptjs": "^2.4.6", "@types/better-sqlite3": "^7.6.12", "@types/node": "^22.13.5", + "@types/ssh2": "^1.15.1", "tsx": "^4.19.3", "typescript": "^5.7.3" } diff --git a/apps/api/src/collectors/sshHost.ts b/apps/api/src/collectors/sshHost.ts new file mode 100644 index 0000000..ad6cb17 --- /dev/null +++ b/apps/api/src/collectors/sshHost.ts @@ -0,0 +1,149 @@ +import { readFileSync } from "node:fs"; +import { Client } from "ssh2"; +import type { Collector, MetricSnapshot } from "./types.js"; + +export interface DiskPath { + path: string; + label: string; +} + +export interface SshHostConfig { + id: string; + displayName: string; + group: string; + host: string; + port: number; + username: string; + privateKeyPath: string; + diskPaths: DiskPath[]; +} + +// Generic Linux boxes Proxmox can't see (bare metal, not LXCs). Pulls the same +// shape of data as ProxmoxCollector via one SSH exec of read-only commands. +// CPU is approximated from 1-min load average / core count since a true +// utilization % needs two /proc/stat samples, not worth a second round trip here. +export class SshHostCollector implements Collector { + readonly id: string; + private readonly privateKey: Buffer; + + constructor(private readonly cfg: SshHostConfig) { + this.id = `ssh:${cfg.id}`; + this.privateKey = readFileSync(cfg.privateKeyPath); + } + + async collect(): Promise { + try { + const output = await this.runRemoteScript(); + return [this.parse(output)]; + } catch (err) { + return [ + { + hostId: `ssh:${this.cfg.id}`, + displayName: this.cfg.displayName, + group: this.cfg.group, + status: "down", + cpuPct: null, + memPct: null, + diskPct: null, + memPressurePct: null, + cpuPressurePct: null, + uptimeSec: null, + meta: { error: err instanceof Error ? err.message : String(err) }, + }, + ]; + } + } + + // The remote authorized_keys entry for this key sets a forced `command=`, so + // whatever we exec here is ignored server-side — the actual read-only script + // lives on the remote host (see docs/ssh-collector-key-setup.md) and must stay + // in sync with `diskPaths` labels below. This is intentional defense in depth: + // a leaked key can only ever run that fixed script, not arbitrary commands. + private runRemoteScript(): Promise { + return new Promise((resolve, reject) => { + const conn = new Client(); + const timeout = setTimeout(() => { + conn.end(); + reject(new Error("ssh timeout")); + }, 10_000); + + conn + .on("ready", () => { + conn.exec("collect-stats", (err, stream) => { + if (err) { + clearTimeout(timeout); + conn.end(); + return reject(err); + } + let stdout = ""; + stream + .on("close", () => { + clearTimeout(timeout); + conn.end(); + resolve(stdout); + }) + .on("data", (data: Buffer) => { + stdout += data.toString(); + }); + }); + }) + .on("error", (err) => { + clearTimeout(timeout); + reject(err); + }) + .connect({ + host: this.cfg.host, + port: this.cfg.port, + username: this.cfg.username, + privateKey: this.privateKey, + readyTimeout: 8_000, + }); + }); + } + + private parse(output: string): MetricSnapshot { + const vars = new Map(); + for (const line of output.split("\n")) { + const eq = line.indexOf("="); + if (eq === -1) continue; + vars.set(line.slice(0, eq), line.slice(eq + 1)); + } + + const nproc = Number(vars.get("NPROC") ?? "1") || 1; + const load1 = Number(vars.get("LOAD1") ?? "0"); + const cpuPct = clamp(round((load1 / nproc) * 100)); + + const [memTotal, memUsed] = (vars.get("MEMLINE") ?? "0:0").split(":").map(Number); + const memPct = memTotal > 0 ? round((memUsed / memTotal) * 100) : null; + + const uptimeSec = Number(vars.get("UPTIME") ?? "0") || null; + + const disks = this.cfg.diskPaths.map((d) => { + const [size, used] = (vars.get(`DISK_${d.label}`) ?? "0:0").split(":").map(Number); + return { label: d.label, pct: size > 0 ? round((used / size) * 100) : null }; + }); + const primaryDiskPct = disks[0]?.pct ?? null; + + return { + hostId: `ssh:${this.cfg.id}`, + displayName: this.cfg.displayName, + group: this.cfg.group, + status: "up", + cpuPct, + memPct, + diskPct: primaryDiskPct, + memPressurePct: null, + cpuPressurePct: null, + uptimeSec, + meta: { disks, loadApproximation: true }, + }; + } +} + +function round(n: number): number { + return Math.round(n * 10) / 10; +} + +function clamp(n: number): number { + return Math.min(Math.max(n, 0), 100); +} diff --git a/apps/api/src/config/index.ts b/apps/api/src/config/index.ts index 15f9e05..6a9fa5d 100644 --- a/apps/api/src/config/index.ts +++ b/apps/api/src/config/index.ts @@ -1,11 +1,23 @@ import { readFileSync } from "node:fs"; import { parse } from "yaml"; +import type { SshHostConfig } from "../collectors/sshHost.js"; + +interface RawSshHost { + id: string; + displayName: string; + group: string; + host: string; + port?: number; + username: string; + diskPaths?: { path: string; label: string }[]; +} export interface HostsConfig { proxmox: { host: string; node: string; }; + sshHosts?: RawSshHost[]; } export interface AppConfig { @@ -26,6 +38,7 @@ export interface AppConfig { tokenId: string; tokenSecret: string; }; + sshHosts: SshHostConfig[]; hosts: HostsConfig; } @@ -37,6 +50,18 @@ function required(name: string): string { export function loadConfig(hostsConfigPath: string): AppConfig { const hosts = parse(readFileSync(hostsConfigPath, "utf8")) as HostsConfig; + const sshPrivateKeyPath = process.env.SSH_PRIVATE_KEY_PATH ?? "./ssh/monitor_ed25519"; + + const sshHosts: SshHostConfig[] = (hosts.sshHosts ?? []).map((h) => ({ + id: h.id, + displayName: h.displayName, + group: h.group, + host: h.host, + port: h.port ?? 22, + username: h.username, + privateKeyPath: sshPrivateKeyPath, + diskPaths: h.diskPaths ?? [{ path: "/", label: "root" }], + })); return { port: Number(process.env.PORT ?? 3000), @@ -54,6 +79,7 @@ export function loadConfig(hostsConfigPath: string): AppConfig { tokenId: required("PROXMOX_TOKEN_ID"), tokenSecret: required("PROXMOX_TOKEN_SECRET"), }, + sshHosts, hosts, }; } diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index b12dc33..d72c418 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -6,6 +6,7 @@ import { loadConfig } from "./config/index.js"; import { openDb, upsertSnapshots, pruneOldSnapshots } from "./db/index.js"; import { seedAdminUser } from "./auth/local.js"; import { ProxmoxCollector } from "./collectors/proxmox.js"; +import { SshHostCollector } from "./collectors/sshHost.js"; import type { Collector } from "./collectors/types.js"; import { registerAuthRoutes } from "./routes/auth.js"; import { registerHostRoutes } from "./routes/hosts.js"; @@ -27,6 +28,7 @@ async function main() { tokenId: cfg.proxmox.tokenId, tokenSecret: cfg.proxmox.tokenSecret, }), + ...cfg.sshHosts.map((h) => new SshHostCollector(h)), ]; async function pollOnce() { diff --git a/config/hosts.yaml b/config/hosts.yaml index 6ff0ed4..30a9c2b 100644 --- a/config/hosts.yaml +++ b/config/hosts.yaml @@ -3,3 +3,25 @@ proxmox: host: 192.168.1.144 node: pve + +# Bare-metal boxes Proxmox can't see. Connects with the dedicated +# `monitor` SSH key (see docs/ssh-collector-key-setup.md), read-only commands only. +sshHosts: + - id: omv + displayName: omv + group: Bare Metal + host: 192.168.1.180 + username: root + diskPaths: + - path: / + label: os + - path: /srv/mergerfs/media + label: media + - id: ripper + displayName: ripper + group: Bare Metal + host: 192.168.1.171 + username: jhodgkin + diskPaths: + - path: / + label: os diff --git a/docker-compose.yml b/docker-compose.yml index 705d890..afd967d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,8 +7,10 @@ services: environment: - HOSTS_CONFIG_PATH=/app/config/hosts.yaml - DB_PATH=/app/data/monitor.db + - SSH_PRIVATE_KEY_PATH=/app/ssh/monitor_ed25519 volumes: - ./config/hosts.yaml:/app/config/hosts.yaml:ro + - ./ssh/monitor_ed25519:/app/ssh/monitor_ed25519:ro - monitor-data:/app/data networks: - web diff --git a/docs/ssh-collector-key-setup.md b/docs/ssh-collector-key-setup.md new file mode 100644 index 0000000..aa94590 --- /dev/null +++ b/docs/ssh-collector-key-setup.md @@ -0,0 +1,35 @@ +# SSH collector key setup + +`SshHostCollector` (`apps/api/src/collectors/sshHost.ts`) monitors boxes Proxmox can't see — +currently `omv` (192.168.1.180) and `ripper` (192.168.1.171). + +## Key + +Dedicated ed25519 keypair, generated on CT122 (not reused from any personal key): +`/opt/homelab-monitor/ssh/monitor_ed25519{,.pub}`, mounted read-only into the `api` container at +`SSH_PRIVATE_KEY_PATH` (see `.env.example`). + +## Forced command (defense in depth) + +Each remote `authorized_keys` entry sets `command="...",no-pty,no-port-forwarding,...` so the +server **ignores whatever command the client requests** and always runs a fixed read-only script +instead. A leaked key can only ever produce these stats — not shell access, not other commands. + +- `omv:/root/.ssh/monitor-readonly.sh` +- `ripper:/home/jhodgkin/.ssh/monitor-readonly.sh` + +Each script emits `NPROC=`, `LOAD1=`, `MEMLINE=total:used`, `UPTIME=`, and one `DISK_