fkrebs/docs
1
0
Fork 0
Code Issues 14 Pull Requests Actions Packages Projects Releases Wiki Activity
Files
b68ab51e0ed4389b989fd21f063244f0aabb2560
docs/services/ingest-pipeline/index.html
T

2236 lines
49 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!doctype html>
<html lang="en" class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<meta name="description" content="Single source of truth for the homelab">
<link rel="canonical" href="http://192.168.1.8:13080/services/ingest-pipeline/">
<link rel="icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
<title>Ingest Pipeline — Filesystem Scan - nuclide.systems docs</title>
<link rel="stylesheet" href="../../assets/stylesheets/main.484c7ddc.min.css">
<link rel="stylesheet" href="../../assets/stylesheets/palette.ab4e12ef.min.css">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
<style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
<script>__md_scope=new URL("../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
</head>
<body dir="ltr" data-md-color-scheme="slate" data-md-color-primary="black" data-md-color-accent="blue">
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" for="__drawer"></label>
<div data-md-component="skip">
<a href="#ingest-pipeline-filesystem-scan" class="md-skip">
Skip to content
</a>
</div>
<div data-md-component="announce">
</div>
<header class="md-header" data-md-component="header">
<nav class="md-header__inner md-grid" aria-label="Header">
<a href="../.." title="nuclide.systems docs" class="md-header__button md-logo" aria-label="nuclide.systems docs" data-md-component="logo">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
</a>
<label class="md-header__button md-icon" for="__drawer">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
</label>
<div class="md-header__title" data-md-component="header-title">
<div class="md-header__ellipsis">
<div class="md-header__topic">
<span class="md-ellipsis">
nuclide.systems docs
</span>
</div>
<div class="md-header__topic" data-md-component="header-topic">
<span class="md-ellipsis">
Ingest Pipeline — Filesystem Scan
</span>
</div>
</div>
</div>
<form class="md-header__option" data-md-component="palette">
<input class="md-option" data-md-color-media="(prefers-color-scheme: dark)" data-md-color-scheme="slate" data-md-color-primary="black" data-md-color-accent="blue" aria-hidden="true" type="radio" name="__palette" id="__palette_0">
<input class="md-option" data-md-color-media="(prefers-color-scheme: light)" data-md-color-scheme="default" data-md-color-primary="black" data-md-color-accent="blue" aria-hidden="true" type="radio" name="__palette" id="__palette_1">
</form>
<script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
<label class="md-header__button md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
</label>
<div class="md-search" data-md-component="search" role="dialog">
<label class="md-search__overlay" for="__search"></label>
<div class="md-search__inner" role="search">
<form class="md-search__form" name="search">
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
<label class="md-search__icon md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
</label>
<nav class="md-search__options" aria-label="Search">
<button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
</button>
</nav>
<div class="md-search__suggest" data-md-component="search-suggest"></div>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
<div class="md-search-result" data-md-component="search-result">
<div class="md-search-result__meta">
Initializing search
</div>
<ol class="md-search-result__list" role="presentation"></ol>
</div>
</div>
</div>
</div>
</div>
</nav>
</header>
<div class="md-container" data-md-component="container">
<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
<div class="md-grid">
<ul class="md-tabs__list">
<li class="md-tabs__item">
<a href="../.." class="md-tabs__link">
Home
</a>
</li>
<li class="md-tabs__item">
<a href="../../CHANGELOG/" class="md-tabs__link">
Changelog
</a>
</li>
<li class="md-tabs__item">
<a href="../../RESUME/" class="md-tabs__link">
Resume
</a>
</li>
<li class="md-tabs__item">
<a href="../../ct-inventory/" class="md-tabs__link">
CT inventory
</a>
</li>
<li class="md-tabs__item">
<a href="../../infra/proxmox-state/" class="md-tabs__link">
Infra
</a>
</li>
<li class="md-tabs__item">
<a href="../homelab-architecture/" class="md-tabs__link">
Services
</a>
</li>
<li class="md-tabs__item">
<a href="../../security/data-leak-audit-comparison/" class="md-tabs__link">
Security & audits
</a>
</li>
<li class="md-tabs__item">
<a href="../../ideas/stack-ideas/" class="md-tabs__link">
Ideas
</a>
</li>
<li class="md-tabs__item">
<a href="../../history/traefik-migration/" class="md-tabs__link">
History
</a>
</li>
</ul>
</div>
</nav>
<main class="md-main" data-md-component="main">
<div class="md-main__inner md-grid">
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary md-nav--lifted" aria-label="Navigation" data-md-level="0">
<label class="md-nav__title" for="__drawer">
<a href="../.." title="nuclide.systems docs" class="md-nav__button md-logo" aria-label="nuclide.systems docs" data-md-component="logo">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
</a>
nuclide.systems docs
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../.." class="md-nav__link">
<span class="md-ellipsis">
Home
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../CHANGELOG/" class="md-nav__link">
<span class="md-ellipsis">
Changelog
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../RESUME/" class="md-nav__link">
<span class="md-ellipsis">
Resume
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../ct-inventory/" class="md-nav__link">
<span class="md-ellipsis">
CT inventory
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_5" >
<label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="0">
<span class="md-ellipsis">
Infra
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_5">
<span class="md-nav__icon md-icon"></span>
Infra
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../infra/proxmox-state/" class="md-nav__link">
<span class="md-ellipsis">
Proxmox state
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../infra/portmap/" class="md-nav__link">
<span class="md-ellipsis">
Port map
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../infra/storage/" class="md-nav__link">
<span class="md-ellipsis">
Storage
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../infra/volumes/" class="md-nav__link">
<span class="md-ellipsis">
Volumes
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../infra/docker-networks/" class="md-nav__link">
<span class="md-ellipsis">
Docker networks
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../infra/connection-hosts/" class="md-nav__link">
<span class="md-ellipsis">
Connection hosts
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../infra/proxmox-memory-audit/" class="md-nav__link">
<span class="md-ellipsis">
Memory audit
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_6" >
<label class="md-nav__link" for="__nav_6" id="__nav_6_label" tabindex="0">
<span class="md-ellipsis">
Services
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_6_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_6">
<span class="md-nav__icon md-icon"></span>
Services
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../homelab-architecture/" class="md-nav__link">
<span class="md-ellipsis">
Homelab architecture
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../dev-environment/" class="md-nav__link">
<span class="md-ellipsis">
Dev environment (Coder + Gitea)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../mcp-gateway/" class="md-nav__link">
<span class="md-ellipsis">
MCP gateway
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../mcp-servers/" class="md-nav__link">
<span class="md-ellipsis">
MCP servers
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../pocket-id/" class="md-nav__link">
<span class="md-ellipsis">
Pocket-ID (OIDC)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../backrest/" class="md-nav__link">
<span class="md-ellipsis">
Backrest (backups)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../databases/" class="md-nav__link">
<span class="md-ellipsis">
Databases
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../secrets-manager/" class="md-nav__link">
<span class="md-ellipsis">
Secrets manager
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../comfyui/" class="md-nav__link">
<span class="md-ellipsis">
ComfyUI
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../zoraxy/" class="md-nav__link">
<span class="md-ellipsis">
Zoraxy
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../adguard-dns/" class="md-nav__link">
<span class="md-ellipsis">
AdGuard DNS
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../cloud-gpu/" class="md-nav__link">
<span class="md-ellipsis">
Cloud GPU
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../arcane/" class="md-nav__link">
<span class="md-ellipsis">
Arcane
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../doc-ingestion/" class="md-nav__link">
<span class="md-ellipsis">
Doc ingestion pipeline
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../llm-benchmark/" class="md-nav__link">
<span class="md-ellipsis">
LLM benchmark
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_7" >
<label class="md-nav__link" for="__nav_7" id="__nav_7_label" tabindex="0">
<span class="md-ellipsis">
Security & audits
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_7_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_7">
<span class="md-nav__icon md-icon"></span>
Security & audits
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../security/data-leak-audit-comparison/" class="md-nav__link">
<span class="md-ellipsis">
Comparison
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../security/data-leak-audit-2026-05-20-tr004-cloud-sandbox/" class="md-nav__link">
<span class="md-ellipsis">
2026-05-20 · cloud-sandbox breach
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../security/data-leak-audit-2026-05-21-tr004-artifacts/" class="md-nav__link">
<span class="md-ellipsis">
2026-05-21 · artifacts (clean)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../security/audit-claude-code-meta/" class="md-nav__link">
<span class="md-ellipsis">
Self-audit (Claude Code)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../security/transcripts/audit-chat-2026-05-21/" class="md-nav__link">
<span class="md-ellipsis">
Transcript (audit session)
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_8" >
<label class="md-nav__link" for="__nav_8" id="__nav_8_label" tabindex="0">
<span class="md-ellipsis">
Ideas
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_8">
<span class="md-nav__icon md-icon"></span>
Ideas
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../ideas/stack-ideas/" class="md-nav__link">
<span class="md-ellipsis">
Stack ideas
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_9" >
<label class="md-nav__link" for="__nav_9" id="__nav_9_label" tabindex="0">
<span class="md-ellipsis">
History
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_9">
<span class="md-nav__icon md-icon"></span>
History
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../history/traefik-migration/" class="md-nav__link">
<span class="md-ellipsis">
Traefik (abandoned 2026-05-16)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../history/traefik-migration-docker-labels/" class="md-nav__link">
<span class="md-ellipsis">
Traefik labels (abandoned)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../history/mcp-gateway-requirements/" class="md-nav__link">
<span class="md-ellipsis">
MCP gateway requirements (superseded)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../history/scrubbing-list-2026-05-17/" class="md-nav__link">
<span class="md-ellipsis">
Scrubbing list (2026-05-17)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../history/case-study/" class="md-nav__link">
<span class="md-ellipsis">
Case study
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#architecture" class="md-nav__link">
<span class="md-ellipsis">
Architecture
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#why-not-n8n-for-this" class="md-nav__link">
<span class="md-ellipsis">
Why not n8n for this
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#supported-file-types" class="md-nav__link">
<span class="md-ellipsis">
Supported file types
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#script-ingestpy" class="md-nav__link">
<span class="md-ellipsis">
Script: ingest.py
</span>
</a>
<nav class="md-nav" aria-label="Script: ingest.py">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#usage" class="md-nav__link">
<span class="md-ellipsis">
Usage
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#state-file" class="md-nav__link">
<span class="md-ellipsis">
State file
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#qdrant-collection-schema" class="md-nav__link">
<span class="md-ellipsis">
Qdrant collection schema
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#chunking-strategy" class="md-nav__link">
<span class="md-ellipsis">
Chunking strategy
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#error-handling" class="md-nav__link">
<span class="md-ellipsis">
Error handling
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#building-and-running" class="md-nav__link">
<span class="md-ellipsis">
Building and running
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#open-webui-integration" class="md-nav__link">
<span class="md-ellipsis">
Open WebUI integration
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#ongoing-ingestion-n8n" class="md-nav__link">
<span class="md-ellipsis">
Ongoing ingestion (n8n)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#paths-available-on-ct-104" class="md-nav__link">
<span class="md-ellipsis">
Paths available on CT 104
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#status" class="md-nav__link">
<span class="md-ellipsis">
Status
</span>
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content" data-md-component="content">
<article class="md-content__inner md-typeset">
<h1 id="ingest-pipeline-filesystem-scan">Ingest Pipeline — Filesystem Scan<a class="headerlink" href="#ingest-pipeline-filesystem-scan" title="Permanent link">&para;</a></h1>
<p>One-shot pipeline that crawls a directory, converts documents to text, embeds them via
the nomic service, and stores them in Qdrant. Run it manually against any mounted path.</p>
<h2 id="architecture">Architecture<a class="headerlink" href="#architecture" title="Permanent link">&para;</a></h2>
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a> ┌─────────────────────────┐
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a> │ ingest.py --path /... │ runs on CT 104 (has direct access to ai-internal network)
<a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a> └──────────┬──────────────┘
<a id="__codelineno-0-4" name="__codelineno-0-4" href="#__codelineno-0-4"></a> │ per file
<a id="__codelineno-0-5" name="__codelineno-0-5" href="#__codelineno-0-5"></a> ▼
<a id="__codelineno-0-6" name="__codelineno-0-6" href="#__codelineno-0-6"></a> ┌────────────────────────────────────────────────────────────────┐
<a id="__codelineno-0-7" name="__codelineno-0-7" href="#__codelineno-0-7"></a> │ 1. File scanner glob recursively, filter by ext │
<a id="__codelineno-0-8" name="__codelineno-0-8" href="#__codelineno-0-8"></a> │ 2. Change detection sha256(path + mtime) → skip if seen │
<a id="__codelineno-0-9" name="__codelineno-0-9" href="#__codelineno-0-9"></a> │ 3. Docling converter POST http://docling:5001/convert │
<a id="__codelineno-0-10" name="__codelineno-0-10" href="#__codelineno-0-10"></a> │ (PDF/DOCX/PPTX/XLSX/HTML → Markdown) │
<a id="__codelineno-0-11" name="__codelineno-0-11" href="#__codelineno-0-11"></a> │ Plain text/Markdown read directly │
<a id="__codelineno-0-12" name="__codelineno-0-12" href="#__codelineno-0-12"></a> │ Images (jpg/png/...) send to nomic vision endpoint │
<a id="__codelineno-0-13" name="__codelineno-0-13" href="#__codelineno-0-13"></a> │ 4. Chunker split Markdown by headers + size │
<a id="__codelineno-0-14" name="__codelineno-0-14" href="#__codelineno-0-14"></a> │ chunk_size=1200 overlap=150 (matches OWUI RAG config) │
<a id="__codelineno-0-15" name="__codelineno-0-15" href="#__codelineno-0-15"></a> │ 5. Nomic embedder POST http://nomic:80/v1/embeddings │
<a id="__codelineno-0-16" name="__codelineno-0-16" href="#__codelineno-0-16"></a> │ text chunks → 768d images → 768d (same space!) │
<a id="__codelineno-0-17" name="__codelineno-0-17" href="#__codelineno-0-17"></a> │ 6. Qdrant upsert collection `documents`, named vectors │
<a id="__codelineno-0-18" name="__codelineno-0-18" href="#__codelineno-0-18"></a> └────────────────────────────────────────────────────────────────┘
<a id="__codelineno-0-19" name="__codelineno-0-19" href="#__codelineno-0-19"></a> │
<a id="__codelineno-0-20" name="__codelineno-0-20" href="#__codelineno-0-20"></a> ▼
<a id="__codelineno-0-21" name="__codelineno-0-21" href="#__codelineno-0-21"></a> ┌──────────────────────────────────────────────────────────────┐
<a id="__codelineno-0-22" name="__codelineno-0-22" href="#__codelineno-0-22"></a> │ Qdrant: http://qdrant:6333 │
<a id="__codelineno-0-23" name="__codelineno-0-23" href="#__codelineno-0-23"></a> │ collection: documents │
<a id="__codelineno-0-24" name="__codelineno-0-24" href="#__codelineno-0-24"></a> │ vector: {size: 768, distance: Cosine} │
<a id="__codelineno-0-25" name="__codelineno-0-25" href="#__codelineno-0-25"></a> │ payload: {path, title, chunk_idx, text, type, mtime, hash} │
<a id="__codelineno-0-26" name="__codelineno-0-26" href="#__codelineno-0-26"></a> └──────────────────────────────────────────────────────────────┘
<a id="__codelineno-0-27" name="__codelineno-0-27" href="#__codelineno-0-27"></a> │
<a id="__codelineno-0-28" name="__codelineno-0-28" href="#__codelineno-0-28"></a> ▼
<a id="__codelineno-0-29" name="__codelineno-0-29" href="#__codelineno-0-29"></a> ┌────────────────────────────┐
<a id="__codelineno-0-30" name="__codelineno-0-30" href="#__codelineno-0-30"></a> │ Open WebUI RAG search │ queries `documents` collection
<a id="__codelineno-0-31" name="__codelineno-0-31" href="#__codelineno-0-31"></a> │ n8n / MCP tools │ same collection, same embedding space
<a id="__codelineno-0-32" name="__codelineno-0-32" href="#__codelineno-0-32"></a> └────────────────────────────┘
</code></pre></div>
<h2 id="why-not-n8n-for-this">Why not n8n for this<a class="headerlink" href="#why-not-n8n-for-this" title="Permanent link">&para;</a></h2>
<p>n8n is the right tool for <strong>event-driven</strong> pipelines (webhook on new file, Paperless
webhook, Nextcloud activity, scheduled re-sync). For a <strong>bulk one-shot crawl</strong>, a Python
script is better:
- direct filesystem access with <code>os.walk</code>
- proper progress bar and error recovery
- batched Qdrant upserts (n8n does one HTTP call per node)
- easy CLI: <code>python3 ingest.py /mnt/pve/unas/Notizen</code></p>
<p>n8n handles the <em>ongoing</em> layer (see <a href="#ongoing-ingestion-n8n">Ongoing ingestion</a>).</p>
<h2 id="supported-file-types">Supported file types<a class="headerlink" href="#supported-file-types" title="Permanent link">&para;</a></h2>
<table>
<thead>
<tr>
<th>Extension</th>
<th>Handler</th>
<th>Notes</th>
</tr>
</thead>
<tbody>
<tr>
<td><code>.pdf</code></td>
<td>Docling</td>
<td>best quality, preserves tables</td>
</tr>
<tr>
<td><code>.docx</code>, <code>.odt</code></td>
<td>Docling</td>
<td></td>
</tr>
<tr>
<td><code>.pptx</code></td>
<td>Docling</td>
<td>slides → sections</td>
</tr>
<tr>
<td><code>.xlsx</code>, <code>.ods</code></td>
<td>Docling</td>
<td>tables → Markdown</td>
</tr>
<tr>
<td><code>.html</code>, <code>.htm</code></td>
<td>Docling</td>
<td></td>
</tr>
<tr>
<td><code>.md</code>, <code>.txt</code>, <code>.rst</code></td>
<td>direct read</td>
<td>no conversion needed</td>
</tr>
<tr>
<td><code>.jpg</code>, <code>.jpeg</code>, <code>.png</code>, <code>.webp</code>, <code>.gif</code></td>
<td>nomic vision</td>
<td>768d image embedding, no text chunks</td>
</tr>
</tbody>
</table>
<h2 id="script-ingestpy">Script: <code>ingest.py</code><a class="headerlink" href="#script-ingestpy" title="Permanent link">&para;</a></h2>
<p>Lives at <code>/opt/stacks/ai/ingest/ingest.py</code>. Runs directly on CT 104.</p>
<h3 id="usage">Usage<a class="headerlink" href="#usage" title="Permanent link">&para;</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a><span class="c1"># Index everything under a path</span>
<a id="__codelineno-1-2" name="__codelineno-1-2" href="#__codelineno-1-2"></a>python3<span class="w"> </span>/opt/stacks/ai/ingest/ingest.py<span class="w"> </span>--path<span class="w"> </span>/mnt/pve/unas/Notizen
<a id="__codelineno-1-3" name="__codelineno-1-3" href="#__codelineno-1-3"></a>
<a id="__codelineno-1-4" name="__codelineno-1-4" href="#__codelineno-1-4"></a><span class="c1"># Different collection, force re-index</span>
<a id="__codelineno-1-5" name="__codelineno-1-5" href="#__codelineno-1-5"></a>python3<span class="w"> </span>/opt/stacks/ai/ingest/ingest.py<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-1-6" name="__codelineno-1-6" href="#__codelineno-1-6"></a><span class="w"> </span>--path<span class="w"> </span>/mnt/pve/unas/Dokumente<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-1-7" name="__codelineno-1-7" href="#__codelineno-1-7"></a><span class="w"> </span>--collection<span class="w"> </span>work-docs<span class="w"> </span><span class="se">\</span>
<a id="__codelineno-1-8" name="__codelineno-1-8" href="#__codelineno-1-8"></a><span class="w"> </span>--force
<a id="__codelineno-1-9" name="__codelineno-1-9" href="#__codelineno-1-9"></a>
<a id="__codelineno-1-10" name="__codelineno-1-10" href="#__codelineno-1-10"></a><span class="c1"># Dry run (print files, don&#39;t embed)</span>
<a id="__codelineno-1-11" name="__codelineno-1-11" href="#__codelineno-1-11"></a>python3<span class="w"> </span>/opt/stacks/ai/ingest/ingest.py<span class="w"> </span>--path<span class="w"> </span>/path/to/docs<span class="w"> </span>--dry-run
</code></pre></div>
<h3 id="state-file">State file<a class="headerlink" href="#state-file" title="Permanent link">&para;</a></h3>
<p><code>/opt/stacks/ai/ingest/state/&lt;collection&gt;.json</code> tracks <code>{path: {hash, indexed_at}}</code>.
Subsequent runs skip unchanged files. Delete the state file to force full re-index.</p>
<h3 id="qdrant-collection-schema">Qdrant collection schema<a class="headerlink" href="#qdrant-collection-schema" title="Permanent link">&para;</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-2-1" name="__codelineno-2-1" href="#__codelineno-2-1"></a><span class="n">vectors_config</span> <span class="o">=</span> <span class="n">VectorParams</span><span class="p">(</span><span class="n">size</span><span class="o">=</span><span class="mi">768</span><span class="p">,</span> <span class="n">distance</span><span class="o">=</span><span class="n">Distance</span><span class="o">.</span><span class="n">COSINE</span><span class="p">)</span>
<a id="__codelineno-2-2" name="__codelineno-2-2" href="#__codelineno-2-2"></a><span class="c1"># payload per point:</span>
<a id="__codelineno-2-3" name="__codelineno-2-3" href="#__codelineno-2-3"></a><span class="p">{</span>
<a id="__codelineno-2-4" name="__codelineno-2-4" href="#__codelineno-2-4"></a> <span class="s2">&quot;path&quot;</span><span class="p">:</span> <span class="s2">&quot;/mnt/pve/unas/Notizen/someFile.md&quot;</span><span class="p">,</span>
<a id="__codelineno-2-5" name="__codelineno-2-5" href="#__codelineno-2-5"></a> <span class="s2">&quot;title&quot;</span><span class="p">:</span> <span class="s2">&quot;someFile&quot;</span><span class="p">,</span> <span class="c1"># filename without ext</span>
<a id="__codelineno-2-6" name="__codelineno-2-6" href="#__codelineno-2-6"></a> <span class="s2">&quot;chunk_idx&quot;</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span> <span class="c1"># 0-based chunk index within file</span>
<a id="__codelineno-2-7" name="__codelineno-2-7" href="#__codelineno-2-7"></a> <span class="s2">&quot;total_chunks&quot;</span><span class="p">:</span> <span class="mi">3</span><span class="p">,</span>
<a id="__codelineno-2-8" name="__codelineno-2-8" href="#__codelineno-2-8"></a> <span class="s2">&quot;text&quot;</span><span class="p">:</span> <span class="s2">&quot;…chunk content…&quot;</span><span class="p">,</span> <span class="c1"># empty string for images</span>
<a id="__codelineno-2-9" name="__codelineno-2-9" href="#__codelineno-2-9"></a> <span class="s2">&quot;type&quot;</span><span class="p">:</span> <span class="s2">&quot;markdown&quot;</span><span class="p">,</span> <span class="c1"># markdown | pdf | docx | image | …</span>
<a id="__codelineno-2-10" name="__codelineno-2-10" href="#__codelineno-2-10"></a> <span class="s2">&quot;mtime&quot;</span><span class="p">:</span> <span class="mf">1716700000.0</span><span class="p">,</span>
<a id="__codelineno-2-11" name="__codelineno-2-11" href="#__codelineno-2-11"></a> <span class="s2">&quot;hash&quot;</span><span class="p">:</span> <span class="s2">&quot;a3f…&quot;</span><span class="p">,</span> <span class="c1"># sha256 of file content</span>
<a id="__codelineno-2-12" name="__codelineno-2-12" href="#__codelineno-2-12"></a> <span class="s2">&quot;source&quot;</span><span class="p">:</span> <span class="s2">&quot;filesystem&quot;</span><span class="p">,</span>
<a id="__codelineno-2-13" name="__codelineno-2-13" href="#__codelineno-2-13"></a><span class="p">}</span>
</code></pre></div>
<h3 id="chunking-strategy">Chunking strategy<a class="headerlink" href="#chunking-strategy" title="Permanent link">&para;</a></h3>
<p>For Markdown output from Docling (and raw .md/.txt):
1. Split on <code>##</code> / <code>###</code> headers first (keep header as first line of chunk)
2. If chunk &gt; 1200 chars, split further on double-newline (<code>\n\n</code>)
3. If still &gt; 1200 chars, hard-split with 150-char overlap</p>
<p>Images: single point per file, no chunking.</p>
<h3 id="error-handling">Error handling<a class="headerlink" href="#error-handling" title="Permanent link">&para;</a></h3>
<ul>
<li>Docling timeout (&gt;60s): skip file, log to <code>ingest_errors.log</code>, continue</li>
<li>Qdrant upsert failure: retry 3×, then log and continue</li>
<li>Re-run is safe: state file prevents double-indexing</li>
</ul>
<h2 id="building-and-running">Building and running<a class="headerlink" href="#building-and-running" title="Permanent link">&para;</a></h2>
<div class="highlight"><pre><span></span><code><a id="__codelineno-3-1" name="__codelineno-3-1" href="#__codelineno-3-1"></a><span class="c1"># On CT 104</span>
<a id="__codelineno-3-2" name="__codelineno-3-2" href="#__codelineno-3-2"></a>mkdir<span class="w"> </span>-p<span class="w"> </span>/opt/stacks/ai/ingest
<a id="__codelineno-3-3" name="__codelineno-3-3" href="#__codelineno-3-3"></a><span class="nb">cd</span><span class="w"> </span>/opt/stacks/ai/ingest
<a id="__codelineno-3-4" name="__codelineno-3-4" href="#__codelineno-3-4"></a>
<a id="__codelineno-3-5" name="__codelineno-3-5" href="#__codelineno-3-5"></a><span class="c1"># Install deps (lightweight — no torch needed, calls services via HTTP)</span>
<a id="__codelineno-3-6" name="__codelineno-3-6" href="#__codelineno-3-6"></a>pip3<span class="w"> </span>install<span class="w"> </span>qdrant-client<span class="w"> </span>requests<span class="w"> </span>tqdm
<a id="__codelineno-3-7" name="__codelineno-3-7" href="#__codelineno-3-7"></a>
<a id="__codelineno-3-8" name="__codelineno-3-8" href="#__codelineno-3-8"></a><span class="c1"># Run</span>
<a id="__codelineno-3-9" name="__codelineno-3-9" href="#__codelineno-3-9"></a>python3<span class="w"> </span>ingest.py<span class="w"> </span>--path<span class="w"> </span>/mnt/pve/unas/Notizen
</code></pre></div>
<p>No container needed for the script itself — it runs on CT 104 bare Python and calls
<code>http://docling:5001</code>, <code>http://nomic:80</code>, <code>http://qdrant:6333</code> via <code>ai-internal</code> (all
on the same Docker network/host).</p>
<p>To run it from <em>outside</em> CT 104 (e.g. the PVE host), wrap it in a container later.</p>
<h2 id="open-webui-integration">Open WebUI integration<a class="headerlink" href="#open-webui-integration" title="Permanent link">&para;</a></h2>
<p>OWUI's knowledge base already points at Qdrant (<code>VECTOR_DB=qdrant</code>). To surface
documents from the <code>documents</code> collection in chat:</p>
<ol>
<li>Admin → Knowledge → Create Knowledge Base</li>
<li>Name: "Local Documents"</li>
<li>The collection is populated by <code>ingest.py</code> — OWUI will search it on <code>#</code>-prefixed
RAG queries or when the knowledge base is enabled in a chat.</li>
</ol>
<blockquote>
<p><strong>Note</strong>: OWUI creates its own internal collection names. To share the same <code>documents</code>
collection between <code>ingest.py</code> and OWUI, use the OWUI API to create a knowledge base
pointing to the pre-populated collection — or let OWUI manage its own collection and
have <code>ingest.py</code> add documents via the OWUI knowledge API (<code>POST /api/v1/knowledge/{id}/file/add</code>).
The OWUI API path is cleaner for OWUI search integration; the direct Qdrant path is
better for external tools (n8n, MCP).</p>
</blockquote>
<h2 id="ongoing-ingestion-n8n">Ongoing ingestion (n8n)<a class="headerlink" href="#ongoing-ingestion-n8n" title="Permanent link">&para;</a></h2>
<p>After the one-shot crawl, wire n8n for continuous ingestion:</p>
<table>
<thead>
<tr>
<th>Trigger</th>
<th>n8n nodes</th>
<th>Notes</th>
</tr>
</thead>
<tbody>
<tr>
<td>Nextcloud webhook (file created/modified)</td>
<td>HTTP → SSH → <code>ingest.py --path &lt;file&gt;</code></td>
<td>Nextcloud admin → Webhooks app</td>
</tr>
<tr>
<td>Paperless post-consume webhook</td>
<td>HTTP → Docling → nomic → Qdrant upsert</td>
<td>Paperless has <code>POST_CONSUME_SCRIPT</code> hook</td>
</tr>
<tr>
<td>Cron re-scan</td>
<td>Schedule → SSH → <code>ingest.py --path /mnt/pve/unas/Notizen</code></td>
<td>weekly full re-sync</td>
</tr>
</tbody>
</table>
<p>The cron re-scan is safe because <code>ingest.py</code> skips unchanged files via state hash.</p>
<h2 id="paths-available-on-ct-104">Paths available on CT 104<a class="headerlink" href="#paths-available-on-ct-104" title="Permanent link">&para;</a></h2>
<table>
<thead>
<tr>
<th>Path</th>
<th>Contents</th>
</tr>
</thead>
<tbody>
<tr>
<td><code>/mnt/pve/unas/Notizen/</code></td>
<td>Obsidian vault (Markdown)</td>
</tr>
<tr>
<td><code>/mnt/pve/unas/</code></td>
<td>full UNAS NFS share</td>
</tr>
<tr>
<td><code>/opt/stacks/*/</code></td>
<td>stack configs (already in Git, lower priority)</td>
</tr>
</tbody>
</table>
<p>Nextcloud files are on CT 105 (<code>192.168.1.41</code>). Access via:
- WebDAV: <code>https://nc.nuclide.systems/remote.php/dav/files/fkrebs@nucli.de/</code>
- Or mount the NC data volume — not currently mounted on CT 104</p>
<h2 id="status">Status<a class="headerlink" href="#status" title="Permanent link">&para;</a></h2>
<p><strong>Not yet implemented.</strong> Design only. Next step: write <code>ingest.py</code>.</p>
</article>
</div>
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
</div>
<button type="button" class="md-top md-icon" data-md-component="top" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8z"/></svg>
Back to top
</button>
</main>
<footer class="md-footer">
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-copyright">
Made with
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
Material for MkDocs
</a>
</div>
</div>
</div>
</footer>
</div>
<div class="md-dialog" data-md-component="dialog">
<div class="md-dialog__inner md-typeset"></div>
</div>
<script id="__config" type="application/json">{"annotate": null, "base": "../..", "features": ["navigation.tabs", "navigation.sections", "navigation.expand", "navigation.top", "search.highlight", "search.suggest", "content.code.copy"], "search": "../../assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
<script src="../../assets/javascripts/bundle.79ae519e.min.js"></script>
</body>
</html>
Reference in New Issue View Git Blame Copy Permalink