Files
docs/CHANGELOG/index.html
T

2821 lines
76 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!doctype html>
<html lang="en" class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<meta name="description" content="Single source of truth for the homelab">
<link rel="canonical" href="http://192.168.1.8:13080/CHANGELOG/">
<link rel="prev" href="..">
<link rel="next" href="../RESUME/">
<link rel="icon" href="../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
<title>Changelog - nuclide.systems docs</title>
<link rel="stylesheet" href="../assets/stylesheets/main.484c7ddc.min.css">
<link rel="stylesheet" href="../assets/stylesheets/palette.ab4e12ef.min.css">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
<style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
<script>__md_scope=new URL("..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
</head>
<body dir="ltr" data-md-color-scheme="slate" data-md-color-primary="black" data-md-color-accent="blue">
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" for="__drawer"></label>
<div data-md-component="skip">
<a href="#changelog" class="md-skip">
Skip to content
</a>
</div>
<div data-md-component="announce">
</div>
<header class="md-header" data-md-component="header">
<nav class="md-header__inner md-grid" aria-label="Header">
<a href=".." title="nuclide.systems docs" class="md-header__button md-logo" aria-label="nuclide.systems docs" data-md-component="logo">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
</a>
<label class="md-header__button md-icon" for="__drawer">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
</label>
<div class="md-header__title" data-md-component="header-title">
<div class="md-header__ellipsis">
<div class="md-header__topic">
<span class="md-ellipsis">
nuclide.systems docs
</span>
</div>
<div class="md-header__topic" data-md-component="header-topic">
<span class="md-ellipsis">
Changelog
</span>
</div>
</div>
</div>
<form class="md-header__option" data-md-component="palette">
<input class="md-option" data-md-color-media="(prefers-color-scheme: dark)" data-md-color-scheme="slate" data-md-color-primary="black" data-md-color-accent="blue" aria-hidden="true" type="radio" name="__palette" id="__palette_0">
<input class="md-option" data-md-color-media="(prefers-color-scheme: light)" data-md-color-scheme="default" data-md-color-primary="black" data-md-color-accent="blue" aria-hidden="true" type="radio" name="__palette" id="__palette_1">
</form>
<script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
<label class="md-header__button md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
</label>
<div class="md-search" data-md-component="search" role="dialog">
<label class="md-search__overlay" for="__search"></label>
<div class="md-search__inner" role="search">
<form class="md-search__form" name="search">
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
<label class="md-search__icon md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
</label>
<nav class="md-search__options" aria-label="Search">
<button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
</button>
</nav>
<div class="md-search__suggest" data-md-component="search-suggest"></div>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
<div class="md-search-result" data-md-component="search-result">
<div class="md-search-result__meta">
Initializing search
</div>
<ol class="md-search-result__list" role="presentation"></ol>
</div>
</div>
</div>
</div>
</div>
</nav>
</header>
<div class="md-container" data-md-component="container">
<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
<div class="md-grid">
<ul class="md-tabs__list">
<li class="md-tabs__item">
<a href=".." class="md-tabs__link">
Home
</a>
</li>
<li class="md-tabs__item md-tabs__item--active">
<a href="./" class="md-tabs__link">
Changelog
</a>
</li>
<li class="md-tabs__item">
<a href="../RESUME/" class="md-tabs__link">
Resume
</a>
</li>
<li class="md-tabs__item">
<a href="../ct-inventory/" class="md-tabs__link">
CT inventory
</a>
</li>
<li class="md-tabs__item">
<a href="../infra/proxmox-state/" class="md-tabs__link">
Infra
</a>
</li>
<li class="md-tabs__item">
<a href="../services/homelab-architecture/" class="md-tabs__link">
Services
</a>
</li>
<li class="md-tabs__item">
<a href="../security/data-leak-audit-comparison/" class="md-tabs__link">
Security & audits
</a>
</li>
<li class="md-tabs__item">
<a href="../ideas/stack-ideas/" class="md-tabs__link">
Ideas
</a>
</li>
<li class="md-tabs__item">
<a href="../history/traefik-migration/" class="md-tabs__link">
History
</a>
</li>
</ul>
</div>
</nav>
<main class="md-main" data-md-component="main">
<div class="md-main__inner md-grid">
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary md-nav--lifted" aria-label="Navigation" data-md-level="0">
<label class="md-nav__title" for="__drawer">
<a href=".." title="nuclide.systems docs" class="md-nav__button md-logo" aria-label="nuclide.systems docs" data-md-component="logo">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
</a>
nuclide.systems docs
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href=".." class="md-nav__link">
<span class="md-ellipsis">
Home
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--active">
<input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
<label class="md-nav__link md-nav__link--active" for="__toc">
<span class="md-ellipsis">
Changelog
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<a href="./" class="md-nav__link md-nav__link--active">
<span class="md-ellipsis">
Changelog
</span>
</a>
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#2026-05-23" class="md-nav__link">
<span class="md-ellipsis">
2026-05-23
</span>
</a>
<nav class="md-nav" aria-label="2026-05-23">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#observability-stack-ct-109-ops" class="md-nav__link">
<span class="md-ellipsis">
Observability stack (CT 109 "ops")
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#homarr-ct-109" class="md-nav__link">
<span class="md-ellipsis">
Homarr (CT 109)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#pocket-id-oidc-wired-via-api" class="md-nav__link">
<span class="md-ellipsis">
Pocket-ID OIDC (wired via API)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#arcane-dozzle-migrated-to-ct-109-headless-agents-on-all-hosts" class="md-nav__link">
<span class="md-ellipsis">
Arcane + Dozzle migrated to CT 109; headless agents on all hosts
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#haos-kvm-oom-protection" class="md-nav__link">
<span class="md-ellipsis">
HAOS KVM OOM protection
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#unas-pro-config-tracking" class="md-nav__link">
<span class="md-ellipsis">
UNAS Pro config tracking
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#pve-host-optimizations" class="md-nav__link">
<span class="md-ellipsis">
PVE host optimizations
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#n8n-flow-tracking" class="md-nav__link">
<span class="md-ellipsis">
n8n flow tracking
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#ct-104-ai-repo-cleanup" class="md-nav__link">
<span class="md-ellipsis">
CT 104 ai repo cleanup
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#docs" class="md-nav__link">
<span class="md-ellipsis">
Docs
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#2026-05-22" class="md-nav__link">
<span class="md-ellipsis">
2026-05-22
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#2026-05-21" class="md-nav__link">
<span class="md-ellipsis">
2026-05-21
</span>
</a>
<nav class="md-nav" aria-label="2026-05-21">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#docs-docs-infrastructure" class="md-nav__link">
<span class="md-ellipsis">
Docs &amp; docs infrastructure
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#storage-s3" class="md-nav__link">
<span class="md-ellipsis">
Storage / S3
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#mcp-surface" class="md-nav__link">
<span class="md-ellipsis">
MCP surface
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#dev-environment" class="md-nav__link">
<span class="md-ellipsis">
Dev environment
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#gitea-repos" class="md-nav__link">
<span class="md-ellipsis">
Gitea repos
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#planned" class="md-nav__link">
<span class="md-ellipsis">
Planned
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#2026-05-20" class="md-nav__link">
<span class="md-ellipsis">
2026-05-20
</span>
</a>
<nav class="md-nav" aria-label="2026-05-20">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#auth-identity-single-sign-on-lockdown" class="md-nav__link">
<span class="md-ellipsis">
Auth / Identity (single sign-on lockdown)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#dev-environment-new-ct-111-dev-192168142" class="md-nav__link">
<span class="md-ellipsis">
Dev environment (new CT 111 "dev", 192.168.1.42)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#reverse-proxy" class="md-nav__link">
<span class="md-ellipsis">
Reverse proxy
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#mcp-gateway" class="md-nav__link">
<span class="md-ellipsis">
MCP gateway
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#decommissions" class="md-nav__link">
<span class="md-ellipsis">
Decommissions
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#docs-governance" class="md-nav__link">
<span class="md-ellipsis">
Docs / governance
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#2026-05-19" class="md-nav__link">
<span class="md-ellipsis">
2026-05-19
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#2026-05-17" class="md-nav__link">
<span class="md-ellipsis">
2026-05-17
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#2026-05-16" class="md-nav__link">
<span class="md-ellipsis">
2026-05-16
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../RESUME/" class="md-nav__link">
<span class="md-ellipsis">
Resume
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../ct-inventory/" class="md-nav__link">
<span class="md-ellipsis">
CT inventory
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_5" >
<label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="0">
<span class="md-ellipsis">
Infra
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_5">
<span class="md-nav__icon md-icon"></span>
Infra
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../infra/proxmox-state/" class="md-nav__link">
<span class="md-ellipsis">
Proxmox state
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../infra/portmap/" class="md-nav__link">
<span class="md-ellipsis">
Port map
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../infra/storage/" class="md-nav__link">
<span class="md-ellipsis">
Storage
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../infra/volumes/" class="md-nav__link">
<span class="md-ellipsis">
Volumes
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../infra/docker-networks/" class="md-nav__link">
<span class="md-ellipsis">
Docker networks
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../infra/connection-hosts/" class="md-nav__link">
<span class="md-ellipsis">
Connection hosts
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../infra/proxmox-memory-audit/" class="md-nav__link">
<span class="md-ellipsis">
Memory audit
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_6" >
<label class="md-nav__link" for="__nav_6" id="__nav_6_label" tabindex="0">
<span class="md-ellipsis">
Services
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_6_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_6">
<span class="md-nav__icon md-icon"></span>
Services
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../services/homelab-architecture/" class="md-nav__link">
<span class="md-ellipsis">
Homelab architecture
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../services/dev-environment/" class="md-nav__link">
<span class="md-ellipsis">
Dev environment (Coder + Gitea)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../services/mcp-gateway/" class="md-nav__link">
<span class="md-ellipsis">
MCP gateway
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../services/mcp-servers/" class="md-nav__link">
<span class="md-ellipsis">
MCP servers
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../services/pocket-id/" class="md-nav__link">
<span class="md-ellipsis">
Pocket-ID (OIDC)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../services/backrest/" class="md-nav__link">
<span class="md-ellipsis">
Backrest (backups)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../services/databases/" class="md-nav__link">
<span class="md-ellipsis">
Databases
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../services/secrets-manager/" class="md-nav__link">
<span class="md-ellipsis">
Secrets manager
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../services/comfyui/" class="md-nav__link">
<span class="md-ellipsis">
ComfyUI
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../services/zoraxy/" class="md-nav__link">
<span class="md-ellipsis">
Zoraxy
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../services/adguard-dns/" class="md-nav__link">
<span class="md-ellipsis">
AdGuard DNS
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../services/cloud-gpu/" class="md-nav__link">
<span class="md-ellipsis">
Cloud GPU
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../services/arcane/" class="md-nav__link">
<span class="md-ellipsis">
Arcane
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../services/doc-ingestion/" class="md-nav__link">
<span class="md-ellipsis">
Doc ingestion pipeline
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../services/llm-benchmark/" class="md-nav__link">
<span class="md-ellipsis">
LLM benchmark
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_7" >
<label class="md-nav__link" for="__nav_7" id="__nav_7_label" tabindex="0">
<span class="md-ellipsis">
Security & audits
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_7_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_7">
<span class="md-nav__icon md-icon"></span>
Security & audits
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../security/data-leak-audit-comparison/" class="md-nav__link">
<span class="md-ellipsis">
Comparison
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../security/data-leak-audit-2026-05-20-tr004-cloud-sandbox/" class="md-nav__link">
<span class="md-ellipsis">
2026-05-20 · cloud-sandbox breach
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../security/data-leak-audit-2026-05-21-tr004-artifacts/" class="md-nav__link">
<span class="md-ellipsis">
2026-05-21 · artifacts (clean)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../security/audit-claude-code-meta/" class="md-nav__link">
<span class="md-ellipsis">
Self-audit (Claude Code)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../security/transcripts/audit-chat-2026-05-21/" class="md-nav__link">
<span class="md-ellipsis">
Transcript (audit session)
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_8" >
<label class="md-nav__link" for="__nav_8" id="__nav_8_label" tabindex="0">
<span class="md-ellipsis">
Ideas
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_8">
<span class="md-nav__icon md-icon"></span>
Ideas
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../ideas/stack-ideas/" class="md-nav__link">
<span class="md-ellipsis">
Stack ideas
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_9" >
<label class="md-nav__link" for="__nav_9" id="__nav_9_label" tabindex="0">
<span class="md-ellipsis">
History
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_9">
<span class="md-nav__icon md-icon"></span>
History
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../history/traefik-migration/" class="md-nav__link">
<span class="md-ellipsis">
Traefik (abandoned 2026-05-16)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../history/traefik-migration-docker-labels/" class="md-nav__link">
<span class="md-ellipsis">
Traefik labels (abandoned)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../history/mcp-gateway-requirements/" class="md-nav__link">
<span class="md-ellipsis">
MCP gateway requirements (superseded)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../history/scrubbing-list-2026-05-17/" class="md-nav__link">
<span class="md-ellipsis">
Scrubbing list (2026-05-17)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../history/case-study/" class="md-nav__link">
<span class="md-ellipsis">
Case study
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#2026-05-23" class="md-nav__link">
<span class="md-ellipsis">
2026-05-23
</span>
</a>
<nav class="md-nav" aria-label="2026-05-23">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#observability-stack-ct-109-ops" class="md-nav__link">
<span class="md-ellipsis">
Observability stack (CT 109 "ops")
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#homarr-ct-109" class="md-nav__link">
<span class="md-ellipsis">
Homarr (CT 109)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#pocket-id-oidc-wired-via-api" class="md-nav__link">
<span class="md-ellipsis">
Pocket-ID OIDC (wired via API)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#arcane-dozzle-migrated-to-ct-109-headless-agents-on-all-hosts" class="md-nav__link">
<span class="md-ellipsis">
Arcane + Dozzle migrated to CT 109; headless agents on all hosts
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#haos-kvm-oom-protection" class="md-nav__link">
<span class="md-ellipsis">
HAOS KVM OOM protection
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#unas-pro-config-tracking" class="md-nav__link">
<span class="md-ellipsis">
UNAS Pro config tracking
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#pve-host-optimizations" class="md-nav__link">
<span class="md-ellipsis">
PVE host optimizations
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#n8n-flow-tracking" class="md-nav__link">
<span class="md-ellipsis">
n8n flow tracking
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#ct-104-ai-repo-cleanup" class="md-nav__link">
<span class="md-ellipsis">
CT 104 ai repo cleanup
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#docs" class="md-nav__link">
<span class="md-ellipsis">
Docs
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#2026-05-22" class="md-nav__link">
<span class="md-ellipsis">
2026-05-22
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#2026-05-21" class="md-nav__link">
<span class="md-ellipsis">
2026-05-21
</span>
</a>
<nav class="md-nav" aria-label="2026-05-21">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#docs-docs-infrastructure" class="md-nav__link">
<span class="md-ellipsis">
Docs &amp; docs infrastructure
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#storage-s3" class="md-nav__link">
<span class="md-ellipsis">
Storage / S3
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#mcp-surface" class="md-nav__link">
<span class="md-ellipsis">
MCP surface
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#dev-environment" class="md-nav__link">
<span class="md-ellipsis">
Dev environment
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#gitea-repos" class="md-nav__link">
<span class="md-ellipsis">
Gitea repos
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#planned" class="md-nav__link">
<span class="md-ellipsis">
Planned
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#2026-05-20" class="md-nav__link">
<span class="md-ellipsis">
2026-05-20
</span>
</a>
<nav class="md-nav" aria-label="2026-05-20">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#auth-identity-single-sign-on-lockdown" class="md-nav__link">
<span class="md-ellipsis">
Auth / Identity (single sign-on lockdown)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#dev-environment-new-ct-111-dev-192168142" class="md-nav__link">
<span class="md-ellipsis">
Dev environment (new CT 111 "dev", 192.168.1.42)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#reverse-proxy" class="md-nav__link">
<span class="md-ellipsis">
Reverse proxy
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#mcp-gateway" class="md-nav__link">
<span class="md-ellipsis">
MCP gateway
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#decommissions" class="md-nav__link">
<span class="md-ellipsis">
Decommissions
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#docs-governance" class="md-nav__link">
<span class="md-ellipsis">
Docs / governance
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#2026-05-19" class="md-nav__link">
<span class="md-ellipsis">
2026-05-19
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#2026-05-17" class="md-nav__link">
<span class="md-ellipsis">
2026-05-17
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#2026-05-16" class="md-nav__link">
<span class="md-ellipsis">
2026-05-16
</span>
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content" data-md-component="content">
<article class="md-content__inner md-typeset">
<h1 id="changelog">Changelog<a class="headerlink" href="#changelog" title="Permanent link">&para;</a></h1>
<p>All notable infrastructure / service / doc changes. Newest first.</p>
<h2 id="2026-05-23">2026-05-23<a class="headerlink" href="#2026-05-23" title="Permanent link">&para;</a></h2>
<h3 id="observability-stack-ct-109-ops">Observability stack (CT 109 "ops")<a class="headerlink" href="#observability-stack-ct-109-ops" title="Permanent link">&para;</a></h3>
<ul>
<li>
<p><strong>CT 109 ("ops") fully provisioned</strong>: Debian 13, Docker 29.5.2. Stack <code>/opt/stacks/monitoring/</code> running Prometheus (:9090), Grafana (:3000 LAN-only, admin/tapirnase), node-exporter, Loki, Alloy, pve-exporter, and Gotify-bridge. CT info updated in <code>ct-inventory.md</code>.</p>
</li>
<li>
<p><strong>Loki deployed on CT 109</strong>: log aggregation at <code>:3100</code>, 30-day retention, TSDB v13 schema, filesystem storage at <code>./loki/data/</code>. <code>limits_config</code> set <code>reject_old_samples_max_age: 168h</code> (7 days) to handle log backfill from HA restart. Grafana datasource auto-provisioned via <code>provisioning/datasources/loki.yml</code>. Grafana Loki datasource UID: <code>P8E80F9AEF21F6940</code>.</p>
</li>
<li>
<p><strong>Prometheus remote_write receiver enabled</strong>: <code>--web.enable-remote-write-receiver</code> added to CT 109 Prometheus. Required for HA Alloy add-on to push metrics directly.</p>
</li>
<li>
<p><strong>prometheus-pve-exporter deployed</strong>: <code>prompve/prometheus-pve-exporter:latest</code> on CT 109; credentials in <code>./pve-exporter/pve.yml</code> (chmod 644 — container runs non-root). PVE <code>monitor@pve</code> service user + API token <code>c4be45dc-...</code> created with <code>PVEAuditor</code> role on <code>/</code>. Prometheus scrapes <code>:9221/pve?target=192.168.1.20</code>. Metrics: per-VM/CT CPU, memory, disk, network I/O.</p>
</li>
<li>
<p><strong>HA prometheus integration added</strong>: <code>prometheus:</code> block in HA <code>configuration.yaml</code>; HA exposes ~9211 <code>hass_*</code> entity-level metrics at <code>:8123/api/prometheus</code>. CT 109 Prometheus scrapes with bearer token (HA long-lived token), <code>scrape_interval: 60s</code>. This is distinct from Alloy host-level metrics — both needed.</p>
</li>
<li>
<p><strong>Grafana Alloy deployed across all 12 hosts</strong>: Alloy v1.16.1 from Grafana apt repo. River-syntax configs ship Docker + journal log scraping with host labels to <code>http://192.168.1.8:3100/loki/api/v1/push</code>. Deployment breakdown:</p>
</li>
<li><strong>Docker Compose</strong> (CT 101, 104, 105, 110, 111, 112, 113): container with docker socket + journal mounts, <code>group_add: ["999"]</code> for journal access.</li>
<li><strong>Binary + systemd</strong> (CT 102, 103, 108, nuc): apt install <code>alloy</code>, config at <code>/etc/alloy/config.alloy</code>, <code>systemctl enable --now alloy</code>. Journal-only (no Docker socket on these hosts).</li>
<li><strong>HA add-on</strong> (VM 100): <code>wymangr/hassos-addons</code> Grafana Alloy v0.0.8 — config fills in <code>prometheus_remote_write</code> to CT 109 + Loki endpoint. Distinct host label <code>homeassistant</code>.</li>
<li>
<p>All configs stored in <code>/opt/homelab-configs/alloy/</code> (see homelab-configs repo below).</p>
</li>
<li>
<p><strong>5 Grafana dashboards provisioned</strong>:</p>
</li>
<li>Node Exporter Full (Grafana ID 1860) — imported for all hosts</li>
<li>LiteLLM (custom-built) — <code>/d/.../litellm</code>; panels: request rate, error rate, token throughput, model split, p95 latency</li>
<li>Home Assistant (Grafana ID 12481 + custom panels) — power/energy, climate (temp/humidity/CO₂), home activity, device health/battery</li>
<li>Loki / log explorer — imported</li>
<li>
<p>Prometheus target health — imported</p>
</li>
<li>
<p><strong>3 Grafana alert rules created</strong> via <code>/api/v1/provisioning/alert-rules</code> (folder <code>alerts-folder</code>):</p>
</li>
<li><code>Node disk &gt; 85%</code> — per-instance mountpoint disk usage, 5m window</li>
<li><code>Prometheus target down</code><code>count(up == 0) &gt; 0</code>, 5m window</li>
<li><code>LiteLLM error rate &gt; 5%</code> — 10m rate window, 5m pending</li>
</ul>
<h3 id="homarr-ct-109">Homarr (CT 109)<a class="headerlink" href="#homarr-ct-109" title="Permanent link">&para;</a></h3>
<ul>
<li>
<p><strong>Homarr v1 deployed</strong>: <code>ghcr.io/homarr-labs/homarr:latest</code> on CT 109 <code>:7575</code>. Data at <code>./data/</code>, docker socket bind-mounted read-only for auto-discovery. LAN-only (no Zoraxy route).</p>
</li>
<li>
<p><strong>homelab-configs Gitea repo</strong>: private repo <code>fkrebs/homelab-configs</code> at <code>git.nuclide.systems</code>; cloned to <code>/opt/homelab-configs</code> on CT 109. Daily cron at 03:00 syncs: Alloy configs per host, Homarr docker-compose, Homarr SQLite dump. Script at <code>/usr/local/bin/sync-homelab-configs</code>. Commit + push on any diff.</p>
</li>
</ul>
<h3 id="pocket-id-oidc-wired-via-api">Pocket-ID OIDC (wired via API)<a class="headerlink" href="#pocket-id-oidc-wired-via-api" title="Permanent link">&para;</a></h3>
<ul>
<li>
<p><strong>Homarr OIDC</strong>: client <code>63a94e30</code> created via Pocket-ID API. Redirect URI <code>http://192.168.1.8:7575/api/auth/callback/oidc</code>. Homarr compose updated (<code>AUTH_PROVIDER=oidc</code>, <code>AUTH_OIDC_CLIENT_ID</code>, <code>AUTH_OIDC_CLIENT_SECRET</code>, <code>AUTH_OIDC_ISSUER</code>, etc.), force-recreated. SSO active 2026-05-23.</p>
</li>
<li>
<p><strong>Grafana OIDC</strong>: client <code>92d987d5</code> created via Pocket-ID API with PKCE enabled. Redirect URI <code>http://192.168.1.8:3000/login/generic_oauth</code>. Grafana compose updated with <code>GF_AUTH_GENERIC_OAUTH_*</code> vars, force-recreated. SSO active 2026-05-23; role mapping: <code>admins</code> group → Admin, else Viewer.</p>
</li>
</ul>
<h3 id="arcane-dozzle-migrated-to-ct-109-headless-agents-on-all-hosts">Arcane + Dozzle migrated to CT 109; headless agents on all hosts<a class="headerlink" href="#arcane-dozzle-migrated-to-ct-109-headless-agents-on-all-hosts" title="Permanent link">&para;</a></h3>
<ul>
<li><strong>Arcane server moved CT 104 → CT 109</strong>: stack at <code>/opt/stacks/arcane/docker-compose.yml</code> on CT 109, <code>APP_URL=http://192.168.1.8:10002</code>. Zoraxy proxy route for <code>arcane.nuclide.systems</code> updated to <code>192.168.1.8:10002</code>. Old CT 104 arcane container stopped (renamed decommissioned).</li>
<li><strong>Dozzle server moved CT 104 → CT 109</strong>: stack at <code>/opt/stacks/dozzle/docker-compose.yml</code> (:10001). Remote agents configured for all 7 Docker hosts (CT 101, 104, 105, 110, 111, 112, 113).</li>
<li><strong>Headless agents deployed on CT 101, 104, 105, 110, 111, 112</strong> via <code>/opt/stacks/ops-agents/docker-compose.yml</code> (Arcane agent + Dozzle agent). CT 113 already had Arcane agent; updated <code>MANAGER_API_URL</code> to CT 109 and added Dozzle agent.</li>
<li><strong>docs-server migrated CT 111 → CT 109</strong>: <code>mkdocs.yml</code> site_url fixed, nav expanded. docs-server now at <code>http://192.168.1.8:13080</code>.</li>
</ul>
<h3 id="haos-kvm-oom-protection">HAOS KVM OOM protection<a class="headerlink" href="#haos-kvm-oom-protection" title="Permanent link">&para;</a></h3>
<ul>
<li><strong>Root cause identified</strong>: karakeep_chrome (CT 104, <code>gcr.io/zenika-hub/alpine-chrome:123</code>) triggered host-level OOM. Linux OOM killer chose HAOS KVM (PID 588858, 10.3 GB RSS) as largest victim. HA was not crashing internally — it was being killed externally.</li>
<li><strong>shm_size: '512m' added to karakeep chrome service</strong> and force-recreated to reduce Chrome shared memory pressure.</li>
<li><strong>oom_score_adj=-300 set on HAOS KVM PID</strong> (protects it from OOM killer — minimum score = hardest to kill).</li>
<li><strong>Persistence via systemd timer</strong> (<code>protect-haos-kvm.timer</code> on nuc): fires 60s after boot then every 5 min. Checks if KVM PID for VM 100 has oom_score_adj=-300; sets it if not.</li>
</ul>
<h3 id="unas-pro-config-tracking">UNAS Pro config tracking<a class="headerlink" href="#unas-pro-config-tracking" title="Permanent link">&para;</a></h3>
<ul>
<li><strong>SSH key auth established</strong>: Added nuc's RSA pubkey to UNAS Pro (<code>192.168.1.31</code>) <code>/root/.ssh/authorized_keys</code> via password SSH. Key-based auth now works from nuc without password.</li>
<li><strong><code>fkrebs/unas-conf</code> repo created</strong>: Daily cron (03:20 on nuc) syncs: NFS exports, shares JSON, active exportfs, disk usage, cron config, OS version. Script at <code>/usr/local/bin/sync-unas-conf</code>.</li>
<li><strong>NFSv4 status</strong>: UNAS kernel already supports NFSv4 (<code>+4 +4.1 +4.2</code>) but UniFi OS doesn't configure an <code>fsid=0</code> root export. NFSv4 upgrade (docs #2) remains backlog — would require adding an unmanaged drop-in export that firmware updates could overwrite.</li>
</ul>
<h3 id="pve-host-optimizations">PVE host optimizations<a class="headerlink" href="#pve-host-optimizations" title="Permanent link">&para;</a></h3>
<ul>
<li><strong>LXC startup order set for all CTs</strong>: CT103 (backrest), CT104 (docker), CT105 (nextcloud), CT112 (infisical), CT113 (db) had no startup order defined. Assigned: CT113=4, CT112=6, CT104=7, CT103=8, CT105=9. Full boot order now: DNS(1) → Zoraxy(2) → Pocket-ID(3) → DB(4) → Ops(5) → Infisical(6) → Docker(7) → Backrest(8) → Nextcloud(9) → HAOS(10) → Dev(20). Also set CT109 <code>onboot=1</code> (was missing).</li>
<li><strong>KSM (Kernel Samepage Merging) enabled</strong>: <code>echo 1 &gt; /sys/kernel/mm/ksm/run</code> active now; persisted via <code>/etc/systemd/system/ksm-enable.service</code>. Deduplicates identical pages across LXC containers — effective for shared libc/runtime pages. Zero-risk, online change.</li>
<li><strong>5-minute critical watchdog timer added</strong>: <code>homelab-health-quick.timer</code> on nuc runs <code>QUICK=1 homelab-health</code> every 5 min. Checks: all LXC status (auto-starts if stopped), HAOS KVM (notify only), critical containers (Pocket-ID, Prometheus, Grafana, LiteLLM, MCP gateway). Complements the hourly full sweep.</li>
<li><strong>Gitea issues filed for backlog PVE opts</strong> (<code>fkrebs/docs</code> #2-4): NFS v3→v4 upgrade, ZFS ARC increase to 16 GiB (blocked on CT101 right-sizing), hugepages for HAOS KVM.</li>
</ul>
<h3 id="n8n-flow-tracking">n8n flow tracking<a class="headerlink" href="#n8n-flow-tracking" title="Permanent link">&para;</a></h3>
<ul>
<li><strong>Gitea repo <code>fkrebs/n8n-flows</code> created</strong>: exports all n8n workflows as individual JSON files. Daily cron (CT 104, 03:30) syncs via <code>n8n export:workflow --all</code> → split → commit + push.</li>
<li><strong>IDEAS.md</strong>: 10 automation ideas documented with node designs (health check flow, Backrest notify, Docling pipeline, HA alerts, Gitea digest, WAL-G monitor, etc.).</li>
<li><strong>10 Gitea issues filed</strong> on <code>fkrebs/n8n-flows</code> for each idea — tracked in memory.</li>
<li><strong>Homelab health check (bash/systemd)</strong>: interim implementation on nuc (<code>/usr/local/bin/homelab-health</code>, <code>homelab-health.timer</code>, every 1h). Checks: container status per host via SSH, HA VM status, disk usage, key HTTP endpoints. Auto-fix: <code>docker start</code> for stopped containers, <code>docker prune</code> for CT104 /tmp &gt;80%. Escalates to Gotify (<code>homelab-health</code> app, token <code>Az9NpC-m1jjf571</code>) for crash loops/disk critical. Migration to n8n tracked as issue #1.</li>
</ul>
<h3 id="ct-104-ai-repo-cleanup">CT 104 ai repo cleanup<a class="headerlink" href="#ct-104-ai-repo-cleanup" title="Permanent link">&para;</a></h3>
<ul>
<li><strong>lobehub/data/ untracked</strong>: Postgres data dir was accidentally tracked in git. <code>git rm --cached -r lobehub/data/</code> + added to <code>.gitignore</code>. Committed alongside: litellm-config/config.yaml, mcp-gateway updates (litellm_sync.py, model_assignments.json, models.html, server.py), lobehub.yml (claude-* models).</li>
</ul>
<h3 id="docs">Docs<a class="headerlink" href="#docs" title="Permanent link">&para;</a></h3>
<ul>
<li><strong><code>README.md</code> expanded</strong>: monitoring quick-links table (Homarr, Grafana, Prometheus, Loki, Alloy UI), public services table by category, operator-only LAN links, related repos table (homelab-configs, klipper-config, n8n-flows).</li>
<li><strong><code>infra/portmap.md</code> updated</strong>: added Loki :3100, pve-exporter :9221, Alloy :12345, Homarr :7575; Alloy deployment notes across all 12 hosts; updated monitoring section.</li>
<li><strong><code>services/homelab-architecture.md</code></strong> CT 109 description updated to reflect full stack.</li>
<li><strong><code>services/pocket-id.md</code> created</strong>: OIDC endpoints, client creation walkthrough, per-service env var patterns (Homarr, Grafana, Gitea, Coder, Generic), current client registry, backup notes.</li>
</ul>
<h2 id="2026-05-22">2026-05-22<a class="headerlink" href="#2026-05-22" title="Permanent link">&para;</a></h2>
<ul>
<li>
<p><strong>Pocket-ID backup wired into Backrest</strong>: CT 110 SQLite-only (no Postgres). Pre-backup hook script at <code>/opt/backrest/scripts/pocketid-prestage.sh</code> on CT 103: SSHs to CT 110, runs <code>pocket-id export</code> inside container, <code>docker cp</code>s ZIP to host, scp's ZIP + signing keys to <code>/mnt/pve/unas/services/pocketid-backup/</code>. <code>services-backup-plan</code> snapshots staging → jottacloud. Verified: hook fires before snapshot, snapshot includes Pocket-ID data.</p>
</li>
<li>
<p><strong>Vaultwarden data → local zfs</strong>: Moved <code>/mnt/pve/unas/services/vaultwarden</code> to <code>/opt/stacks/vaultwarden/data</code> (local NVMe). DB was already on CT 113 postgres; only <code>rsa_key.pem</code>, <code>config.json</code>, <code>attachments/</code> remain in the data dir. Stale <code>db.sqlite3</code> deleted. NFS copy at <code>services/vaultwarden/</code> is now orphaned and can be cleaned up.</p>
</li>
<li>
<p><strong>WAL-G backup script fixed (CT 104)</strong>: All nightly postgres backups were failing silently. Three bugs fixed: (1) <code>set -e</code> causing script abort when <code>shared-postgres</code> container not found (it was migrated to CT 113); (2) missing <code>-u postgres</code> on all <code>docker exec</code> calls — WAL-G requires the postgres role; (3) missing explicit data path <code>/var/lib/postgresql/data</code> on <code>backup-push</code>. <code>shared-postgres</code> commented out (CT 113 has its own WAL-G cron). <code>immich_postgres</code> and <code>lobe-postgres</code> now back up successfully to Garage S3.</p>
</li>
<li>
<p><strong>apps/ directory decommissioned (CT 104)</strong>: <code>/opt/stacks/apps/</code> contained dead duplicate compose files for gotify, karakeep, memos, paperless-ngx, traccar, vaultwarden, shared-db — all without <code>.env</code> files, never running. Renamed to <code>.DECOMMISSIONED-2026-05-22</code> following established pattern. The canonical stacks at <code>/opt/stacks/&lt;service&gt;/</code> are unaffected.</p>
</li>
<li>
<p><strong>ComfyUI <code>--async-offload</code> removed</strong>: Flag caused full CPU fallback on GGUF img2img with XPU (3.5 min/step instead of ~7 s/step) and pure-noise output. Removed from <code>ai/comfyui.yml</code>. Current args: <code>--listen 0.0.0.0 --enable-cors-header --use-pytorch-cross-attention --disable-smart-memory --force-fp16</code>. See <code>infra/proxmox-memory-audit.md</code>.</p>
</li>
<li>
<p><strong>CT 105 Nextcloud CIFS → NFS</strong>: Nextcloud was crash-looping ("Appdata is not present") because <code>unas_smb</code> CIFS mount (<code>//192.168.1.31/storage</code>) was not mounted on the host. Root cause: fstab entry was missing <code>_netdev</code>; mount dropped after a host event and was never re-established. Fixed by migrating CT 105 <code>mp0</code> from <code>/mnt/pve/unas_smb</code> to <code>/mnt/pve/unas</code> (same NFS share all other CTs use). CIFS entry removed from fstab. All 9 Nextcloud containers healthy post-restart.</p>
</li>
<li>
<p><strong>arr stack started</strong>: <code>vpn_gluetun</code>, <code>shelfarr</code> (<code>:13004</code>), <code>rdtclient</code> (via VPN, <code>:13001</code>), <code>prowlarr</code> (<code>:13002</code>), <code>audiobookshelf</code> (<code>:13003</code>). Fixed empty <code>labels:</code> in VPN compose.</p>
</li>
<li><strong>claude-max-bridge</strong>: new service on CT 104 (<code>ai-internal</code>, no host port). FastAPI shim wrapping <code>claude</code> CLI as an OpenAI-compatible <code>/v1/chat/completions</code> endpoint. Enables LiteLLM to route to Claude Max subscription at zero API cost. Key details: <code>--strict-mcp-config</code> (no MCP servers), <code>--input-format stream-json</code> (multi-turn), <code>--include-partial-messages</code> (real streaming deltas). Bind-mounts <code>/usr/local/bin/claude</code> + <code>/root/.claude</code> (rw for OAuth token refresh). Source at <code>git.nuclide.systems/fkrebs/claude-max-bridge</code>.</li>
<li><strong>LiteLLM Claude models</strong>: <code>claude-sonnet-4-6</code>, <code>claude-opus-4-7</code>, <code>claude-haiku-4-5</code> + short aliases (<code>sonnet</code>, <code>opus</code>, <code>haiku</code>) + effort variants (<code>claude-sonnet-4-6-high</code>, <code>claude-opus-4-7-high</code>) added via <code>openai/</code> provider pointing at <code>claude-max-bridge:8000/v1</code>. Priced at Anthropic list rates for cost-visibility — actual cost $0 (Max). Bridge features: temperature→effort mapping, <code>extra_body.effort</code> override, <code>extra_body.fallback_model</code>, <code>x_claude_cost_usd</code> and <code>x_claude_rate_limit</code> in responses.</li>
<li><strong>claude-max-bridge v2</strong>: additional CLI parameter mappings: <code>extra_body.system_prompt_mode</code> (<code>replace</code>/<code>append</code><code>--system-prompt</code>/<code>--append-system-prompt</code>), <code>extra_body.max_turns</code><code>--max-turns</code>, <code>extra_body.max_budget_usd</code><code>--max-budget-usd</code>, <code>response_format.json_schema</code><code>--json-schema</code>, <code>extra_body.exclude_dynamic_system_prompt_sections</code><code>--exclude-dynamic-system-prompt-sections</code>. Build context moved from <code>/tmp/</code> to <code>/opt/stacks/ai/claude-max-bridge/</code> (survives reboots). Source committed to Gitea at <code>a781abb</code>.</li>
<li><strong>Shepard MongoDB pinned to 8.0.4</strong>: <code>mongo:8.0</code> (latest) introduced a fatal kernel-version check in 8.0.5+ that rejects Proxmox's <code>7.0.2-5-pve</code> kernel string (parsed as major version 7 ≥ 6.19). Pinned to <code>mongo:8.0.4</code> in <code>/opt/shepard/infrastructure/docker-compose.yml</code>. Track MongoDB bug SERVER-121912 for a patched upstream release before unpinning.</li>
<li><strong>ComfyUI XPU optimisation</strong>: removed <code>--lowvram</code> + <code>--reserve-vram 1.0</code>; added <code>--async-offload</code> (Intel-patched XPU streams, ~10% speedup) + <code>--force-fp16</code>; memory limit 20 G → 24 G (FLUX peaks ~22 GB in shared XPU RAM). <code>--disable-smart-memory</code> kept since <code>get_free_memory()</code> returns the full 58 GB shared pool on Arc — smart eviction is blind to the cgroup limit. See <code>infra/proxmox-memory-audit.md</code>.</li>
<li><strong>WaveSpeed FBC workflow</strong>: <code>ApplyFBCacheOnModel(residual_diff_threshold=0.12)</code> + <code>FluxGuidance(guidance=3.5)</code> + <code>BasicGuider</code> + <code>SamplerCustomAdvanced</code> added to FLUX.1-schnell pipeline. Workflow saved at <code>/mnt/pve/unas/services/comfyui/input/flux-schnell-wavespeed.json</code>. Estimated 3050% additional speedup by caching first-block activations. WaveSpeed FBCache confirmed device-agnostic (no CUDA guards); <code>torch.compile</code> node must NOT be used on XPU.</li>
<li>
<p><strong>LiteLLM benchmark</strong>: <code>bench.py</code> at <code>/opt/stacks/ai/benchmark/</code> measures TTFT + TPS for all configured models using streaming requests; renders xychart diagrams via internal Kroki (<code>http://kroki:8000</code>). First run results: Mistral fastest TTFT (43 ms), Claude bridge competitive at 101143 ms. See <code>services/llm-benchmark.md</code>.</p>
</li>
<li>
<p><strong>MCP gateway</strong>: added <code>git</code>, <code>gitlab</code> (DLR, <code>--pass-environment</code> fix for mcp-proxy env passthrough bug), <code>paper-search</code> (Docker catalog image replaces inline <code>python:3.12-slim</code>); <code>UNPAYWALL_EMAIL</code> wired into paper-search. Removed duplicate <code>papersearch</code> container. Updated <code>services/mcp-gateway.md</code>.</p>
</li>
<li><strong>Nextcloud MCP re-enabled</strong>: <code>cbcoutinho/nextcloud-mcp-server</code> running in <code>single_user_basic</code> mode with <code>NEXTCLOUD_VERIFY_SSL=false</code>. App-password generated non-interactively via <code>occ user:add-app-password</code>. Gateway patched: (1) <code>NEXTCLOUD_VERIFY_SSL</code> / <code>MCP_DEPLOYMENT_MODE</code> now baked into hardcoded SERVERS env; (2) startup overlay extended to persist <code>enabled</code> flag; (3) proxy route bypasses per-user credential provisioning when <code>MCP_DEPLOYMENT_MODE=single_user_basic</code>. 26 servers healthy.</li>
<li><strong>CT 109 renamed <code>observe</code><code>ops</code></strong>: updated <code>ct-inventory.md</code>, <code>homelab-architecture.md</code>, <code>proxmox-state.md</code> (IP changed to <code>.8</code> after conflict resolution).</li>
<li><strong>CT 101 Shepard disk expanded</strong>: 100 GiB → 500 GiB (ZFS online resize, no restart).</li>
<li><strong>CT 112 Infisical marked live</strong>: <code>ct-inventory.md</code> and <code>portmap.md</code> updated; LAN-only, no Zoraxy route.</li>
<li><strong>litellm DB password rotated</strong>: placeholder <code>litellm_password_here</code> replaced in both <code>ai/.env</code> and <code>litellm-config/config.yaml</code> (config.yaml takes precedence over env var). Container restarted, healthy.</li>
<li><strong>UNAS migration dumps cleaned</strong>: 7 × <code>*-migration-20260521.sql</code> (~492 MB) deleted from <code>/mnt/pve/unas/dump/</code>.</li>
<li><strong>Arcane OIDC secret rotated</strong>: new secret updated in Pocket-ID SQLite DB (bcrypt, cost 10) and <code>/opt/stacks/arcane/docker-compose.yml</code>. Container restarted.</li>
<li><strong>n8n OIDC secret rotated</strong>: Pocket-ID DB bcrypt update; secret applied to <code>/opt/stacks/n8n/docker-compose.yml</code> (authoritative stack). <code>/opt/stacks/apps/n8n/</code> was a dead duplicate (no <code>.env</code>, never ran) — compose renamed <code>.DECOMMISSIONED-2026-05-22</code>.</li>
<li><strong>n8n encryption key rotated</strong>: <code>N8N_ENCRYPTION_KEY</code> + <code>N8N_USER_MANAGEMENT_JWT_SECRET</code> replaced with 32-byte hex values in <code>/opt/stacks/n8n/.env</code> and <code>/opt/stacks/n8n/data/config</code>. Container restarted, healthy. Any previously stored workflow credentials are now invalid and must be re-entered.</li>
<li><strong>Backrest repo passwords rotated</strong>: <code>tapirnase</code> (weak placeholder) replaced on both <code>media-repo</code> and <code>services-repo</code> (JottaCloud). Used <code>restic key add</code> + <code>restic key remove</code>; <code>config.json</code> updated; daemon restarted.</li>
<li><strong>Obsidian MCP (<code>mcpvault</code>)</strong>: new server added. <code>bitbonsai/mcpvault</code> bridged via <code>mcp-proxy</code> stdio→HTTP in a custom <code>mcp-obsidian-bridge</code> image (Node 3.12-Alpine + mcp-proxy). Vault: Nextcloud <code>Notizen/</code> folder at <code>/mnt/pve/unas/services/nextcloud/fkrebs@nucli.de/files/Notizen</code>, bind-mounted read-write. 15 tools: read/write/patch/search/tags/frontmatter/stats. TTL cache 30s. Registered in Claude Code settings.json and LobeHub for both users.</li>
<li><strong>LobeHub MCP sync</strong>: all 27 gateway servers synced into <code>user_installed_plugins</code> for both users; stale <code>papersearch</code> entry removed; all manifests updated to long-lived <code>mcp_*</code> gateway token. Skipped: <code>crawl4ai</code> (no <code>/mcp</code> endpoint in <code>unclecode/crawl4ai:latest</code>), <code>n8n</code> (uses own auth).</li>
<li><strong>CT 109 plan updated</strong>: IP conflict noted (<code>.6</code> taken by CT 113); RAM 8 GiB, disk 50 GiB; Loki added; sshwifty added (multi-tab web SSH); Alloy replaces Promtail on all hosts; sidecar table expanded to CT 103/111/113.</li>
<li><strong>Portmap</strong>: CT 113 (<code>db</code>, <code>.6</code>) section added; QNAP TS-251D (Klipper/Mainsail) section added.</li>
<li><strong>New docs</strong>: <code>services/backrest.md</code>, <code>services/databases.md</code>, <code>services/arcane.md</code>.</li>
</ul>
<h2 id="2026-05-21">2026-05-21<a class="headerlink" href="#2026-05-21" title="Permanent link">&para;</a></h2>
<h3 id="docs-docs-infrastructure">Docs &amp; docs infrastructure<a class="headerlink" href="#docs-docs-infrastructure" title="Permanent link">&para;</a></h3>
<ul>
<li><strong>Mermaid diagrams now rendering</strong> in mkdocs Material site — was missing <code>custom_fences</code> config in <code>pymdownx.superfences</code>. Fixed in <code>docs-server/mkdocs.yml</code> and rebuilt container.</li>
<li><strong><code>site_url</code> corrected</strong> to <code>http://192.168.1.111:13080</code> (docs are LAN-only; no Zoraxy route exists).</li>
<li><strong>Docs restructured</strong>: "Stacks (CT 104)" chapter dissolved; PORTMAP, Storage, Volumes, Docker networks moved to <code>infra/</code> section. TODO items migrated into CHANGELOG + ideas.</li>
<li><strong>Secrets manager design doc</strong> added at <code>services/secrets-manager.md</code> — recommends Infisical on a new CT 112.</li>
</ul>
<h3 id="storage-s3">Storage / S3<a class="headerlink" href="#storage-s3" title="Permanent link">&para;</a></h3>
<ul>
<li><strong><code>s3.nuclide.systems</code> Zoraxy route fixed</strong><code>EnableWebsocketCustomHeaders</code>, <code>DisableHopByHopHeaderRemoval</code>, <code>EnableAutoHTTPS</code> all set. Garage S3 API now reachable externally.</li>
<li><strong><code>chat-artifacts</code> Garage S3 bucket created</strong> — public-read via <code>https://chat-artifacts.s3.nuclide.systems/&lt;key&gt;</code>. Credentials at <code>/root/garage-chat-artifacts.creds</code> (not git-tracked).</li>
</ul>
<h3 id="mcp-surface">MCP surface<a class="headerlink" href="#mcp-surface" title="Permanent link">&para;</a></h3>
<ul>
<li><strong><code>upload-artifact</code> MCP server</strong> deployed (port 18011, CT 104). Tools: <code>upload_text</code>, <code>upload_base64</code>, <code>list_artifacts</code>, <code>get_url</code>, <code>delete_artifact</code>. Registered in gateway (now 24 servers). Source: <code>git.nuclide.systems/fkrebs/mcp-upload-artifact</code>.</li>
<li><strong>ComfyUI MCP</strong> now auto-uploads completed images to <code>chat-artifacts</code> S3 and includes the public URL alongside the inline base64 image.</li>
<li><strong>DAYTONA_API_KEY</strong> removed from CT 104 <code>ai/.env</code> (Daytona decommissioned 2026-05-20).</li>
</ul>
<h3 id="dev-environment">Dev environment<a class="headerlink" href="#dev-environment" title="Permanent link">&para;</a></h3>
<ul>
<li><strong><code>savefig</code> dotfiles helper</strong> added to <code>fkrebs/dotfiles</code> — uploads any file to <code>chat-artifacts</code> using <code>uv run --with boto3</code>; reads credentials from env vars.</li>
<li><strong>Coder <code>python-uv</code> template</strong> updated with <code>CHAT_ARTIFACTS_*</code> env vars baked in; template pushed to Coder server.</li>
</ul>
<h3 id="gitea-repos">Gitea repos<a class="headerlink" href="#gitea-repos" title="Permanent link">&para;</a></h3>
<ul>
<li>New repos created: <code>mcp-comfyui</code>, <code>mcp-docling</code>, <code>mcp-shepard</code>, <code>mcp-upload-artifact</code>, <code>home-assistant-config</code> — all at <code>git.nuclide.systems/fkrebs/</code>.</li>
<li><strong>HAOS config</strong> (<code>192.168.1.60</code>, VM 100) has a Gitea upstream ready. Manual push needed from HA terminal (see <code>services/secrets-manager.md</code> for SSH access notes).</li>
</ul>
<h3 id="planned">Planned<a class="headerlink" href="#planned" title="Permanent link">&para;</a></h3>
<ul>
<li>CT 112 "secrets" — Infisical deployment (see <code>services/secrets-manager.md</code>)</li>
<li>Renovate Bot via Gitea Actions — dependency updates for MCP server repos (Dockerfiles + requirements.txt)</li>
<li>Vaultwarden OIDC SSO wiring (client created in Pocket-ID 2026-05-21; auth flow not yet configured)</li>
<li>LobeHub Artifacts → S3 patch (TypeScript, ~3-4h, deferred)</li>
</ul>
<h2 id="2026-05-20">2026-05-20<a class="headerlink" href="#2026-05-20" title="Permanent link">&para;</a></h2>
<h3 id="auth-identity-single-sign-on-lockdown">Auth / Identity (single sign-on lockdown)<a class="headerlink" href="#auth-identity-single-sign-on-lockdown" title="Permanent link">&para;</a></h3>
<ul>
<li><strong>Pocket-ID migrated CT 104 → CT 110 "id"</strong> (<code>192.168.1.5:11000</code>). Authoritative writes now on CT 110; old data dir on CT 104 is a stale snapshot. Compose on CT 104 renamed <code>.MIGRATED-TO-CT110-2026-05-20</code>. Zoraxy upstream for <code>id.nuclide.systems</code> flipped.</li>
<li><strong>Pocket-ID 2.7.0 uses bcrypt for OIDC client secrets</strong> — previously some clients (Gitea, Coder) had raw 64-char SHA-256 hex in <code>oidc_clients.secret</code>, which silently failed every token exchange. Regenerated both with proper bcrypt hashes; audit also surfaced an orphaned Daytona client and the empty-secret <code>vscode</code> row.</li>
<li><strong>Gitea SSO via Pocket-ID</strong> wired with bcrypt fix + correct <code>Scopes</code> (<code>openid,profile,email</code>) + <code>two_factor_policy=skip</code> + <code>ACCOUNT_LINKING=auto</code> + <code>ENABLE_AUTO_REGISTRATION=true</code> + <code>USERNAME=preferred_username</code>. Local password sign-in form disabled (<code>GITEA__service__ENABLE_PASSWORD_SIGNIN_FORM=false</code>); legacy OpenID 2.0 button hidden (<code>GITEA__openid__ENABLE_OPENID_SIGNIN=false</code>).</li>
<li><strong>Coder SSO via Pocket-ID</strong> wired. User row flipped <code>login_type=password → oidc</code>. <code>CODER_DISABLE_PASSWORD_AUTH=true</code> + <code>CODER_OAUTH2_GITHUB_DEFAULT_PROVIDER_ENABLE=false</code>. Workspace-side GitHub external_auth left on (for repo cloning).</li>
</ul>
<h3 id="dev-environment-new-ct-111-dev-192168142">Dev environment (new CT 111 "dev", <code>192.168.1.42</code>)<a class="headerlink" href="#dev-environment-new-ct-111-dev-192168142" title="Permanent link">&para;</a></h3>
<ul>
<li>New LXC stood up with Coder + Gitea + their Postgres companions.</li>
<li>Templates pushed to Coder: <strong><code>python-uv</code></strong> (persistent, GPU, code-server, baked LiteLLM + Anthropic env, Claude Code auto-install) and <strong><code>mcp-sandbox</code></strong> (ephemeral, sci stack pre-baked).</li>
<li>GPU passthrough confirmed: <code>/dev/dri/by-path/pci-0000:00:02.0-render</code> exposed; CT 111-level symlink at <code>/dev/dri/renderD128</code> (systemd-tmpfiles) so the Docker provider's path parser is happy.</li>
<li><code>dotfiles</code> repo created at <code>fkrebs/dotfiles</code> on Gitea — oh-my-zsh, LiteLLM model picker (<code>models.env</code>), docker alias set, <code>~/.claude/settings.json</code> permissions allowlist, global Python <code>CLAUDE.md</code>, <code>.gitignore_global</code>, VSCodium settings + extensions.txt.</li>
<li><code>mkproj</code> helper appends <code>[tool.coder]</code> workspace+owner block to new <code>pyproject.toml</code>, copies pre-commit config + <code>.gitignore</code>, installs dev deps, initial commit.</li>
</ul>
<h3 id="reverse-proxy">Reverse proxy<a class="headerlink" href="#reverse-proxy" title="Permanent link">&para;</a></h3>
<ul>
<li><strong>Zoraxy WebSocket forwarding enabled</strong> for <code>dev.</code>, <code>git.</code>, <code>mcp.nuclide.systems</code> (<code>EnableWebsocketCustomHeaders=true</code> + <code>HeaderRewriteRules.DisableHopByHopHeaderRemoval=true</code>). Coder agent / Gitea live updates / streamable-MCP all need this. Backups at <code>/tmp/zoraxy-backup-2026-05-20/</code> on CT 108.</li>
</ul>
<h3 id="mcp-gateway">MCP gateway<a class="headerlink" href="#mcp-gateway" title="Permanent link">&para;</a></h3>
<ul>
<li><strong><code>daytona</code> removed</strong>, <strong><code>coder</code> added</strong> in <code>/opt/stacks/ai/mcp-gateway/config.json</code>. The <code>morning-briefing</code> scheduled agent's server list updated. Built a small <code>coder-mcp</code> image (multi-stage from <code>ghcr.io/coder/coder:latest</code> + <code>uv:bookworm-slim</code> + <code>uv tool install mcp-proxy</code>) that <code>coder login</code>s once then runs <code>coder exp mcp server</code> behind streamable-HTTP.</li>
<li>Coder MCP exposes tools <code>coder_create_task</code> + workspace lifecycle from <code>Coder Agent v1.27.1</code>.</li>
</ul>
<h3 id="decommissions">Decommissions<a class="headerlink" href="#decommissions" title="Permanent link">&para;</a></h3>
<ul>
<li><strong>Daytona</strong> (<code>/opt/stacks/daytona/</code> + <code>/opt/stacks/ai/daytona-mcp.yml</code> on CT 104) — containers stopped + removed; compose files renamed <code>.DECOMMISSIONED-2026-05-20</code>. Coder replaces it.</li>
</ul>
<h3 id="docs-governance">Docs / governance<a class="headerlink" href="#docs-governance" title="Permanent link">&para;</a></h3>
<ul>
<li><strong><code>/CLAUDE.md</code> created on host</strong> — operator doctrine for any agent SSHing in: keep things running, enforce consistency, plan rollback at medium-risk+, services AI-accessible by default, test before declaring done, generate slash commands for recurring tasks, inventory skills/commands/MCPs at session start.</li>
<li><strong><code>/docs</code> is now a git repo</strong>, baseline at <code>04d54ac</code>.</li>
<li><strong>Doc reorg</strong> into <code>infra/ services/ history/ ideas/</code>. Traefik guides moved to <code>history/</code> (Zoraxy is permanent). <code>mcp-gateway-requirements.md</code> and <code>scrubbing-list-2026-05-17.md</code> also archived.</li>
</ul>
<h2 id="2026-05-19">2026-05-19<a class="headerlink" href="#2026-05-19" title="Permanent link">&para;</a></h2>
<ul>
<li><strong>Garage moved off NFS-on-UNAS to local zfs</strong> on CT 104 (<code>/opt/stacks/shared-db/garage/{meta,data}</code>) after a WAL-G outage. Tier-1 (SQLite) databases off NFS as a follow-up; UNAS reserved for bulk media + workspace home dirs.</li>
<li><strong>n8n reverted to local SQLite</strong> after a Postgres migration attempt failed.</li>
</ul>
<h2 id="2026-05-17">2026-05-17<a class="headerlink" href="#2026-05-17" title="Permanent link">&para;</a></h2>
<ul>
<li>MCP gateway design frozen. Implementation in <code>services/mcp-gateway.md</code>; design rationale archived as <code>history/mcp-gateway-requirements.md</code>.</li>
<li>Per-stack scrubbing pass (see <code>history/scrubbing-list-2026-05-17.md</code> for the inventory at that snapshot).</li>
</ul>
<h2 id="2026-05-16">2026-05-16<a class="headerlink" href="#2026-05-16" title="Permanent link">&para;</a></h2>
<ul>
<li><strong>Traefik abandoned</strong>, <strong>Zoraxy is the production reverse proxy</strong>. Two migration guides preserved in <code>history/</code> for context only.</li>
</ul>
</article>
</div>
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
</div>
<button type="button" class="md-top md-icon" data-md-component="top" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8z"/></svg>
Back to top
</button>
</main>
<footer class="md-footer">
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-copyright">
Made with
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
Material for MkDocs
</a>
</div>
</div>
</div>
</footer>
</div>
<div class="md-dialog" data-md-component="dialog">
<div class="md-dialog__inner md-typeset"></div>
</div>
<script id="__config" type="application/json">{"annotate": null, "base": "..", "features": ["navigation.tabs", "navigation.sections", "navigation.expand", "navigation.top", "search.highlight", "search.suggest", "content.code.copy"], "search": "../assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
<script src="../assets/javascripts/bundle.79ae519e.min.js"></script>
</body>
</html>