fkrebs/docs
1
0
Fork 0
Code Issues 14 Pull Requests Actions Packages Projects Releases Wiki Activity
Files
b68ab51e0ed4389b989fd21f063244f0aabb2560
docs/services/doc-ingestion/index.html
T

2310 lines
46 KiB
HTML
Raw Blame History

<!doctype html>
<html lang="en" class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<meta name="description" content="Single source of truth for the homelab">
<link rel="canonical" href="http://192.168.1.8:13080/services/doc-ingestion/">
<link rel="prev" href="../arcane/">
<link rel="next" href="../llm-benchmark/">
<link rel="icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">
<title>Doc ingestion pipeline - nuclide.systems docs</title>
<link rel="stylesheet" href="../../assets/stylesheets/main.484c7ddc.min.css">
<link rel="stylesheet" href="../../assets/stylesheets/palette.ab4e12ef.min.css">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
<style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
<script>__md_scope=new URL("../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
</head>
<body dir="ltr" data-md-color-scheme="slate" data-md-color-primary="black" data-md-color-accent="blue">
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" for="__drawer"></label>
<div data-md-component="skip">
<a href="#document-ingestion-pipeline" class="md-skip">
Skip to content
</a>
</div>
<div data-md-component="announce">
</div>
<header class="md-header" data-md-component="header">
<nav class="md-header__inner md-grid" aria-label="Header">
<a href="../.." title="nuclide.systems docs" class="md-header__button md-logo" aria-label="nuclide.systems docs" data-md-component="logo">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
</a>
<label class="md-header__button md-icon" for="__drawer">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
</label>
<div class="md-header__title" data-md-component="header-title">
<div class="md-header__ellipsis">
<div class="md-header__topic">
<span class="md-ellipsis">
nuclide.systems docs
</span>
</div>
<div class="md-header__topic" data-md-component="header-topic">
<span class="md-ellipsis">
Doc ingestion pipeline
</span>
</div>
</div>
</div>
<form class="md-header__option" data-md-component="palette">
<input class="md-option" data-md-color-media="(prefers-color-scheme: dark)" data-md-color-scheme="slate" data-md-color-primary="black" data-md-color-accent="blue" aria-hidden="true" type="radio" name="__palette" id="__palette_0">
<input class="md-option" data-md-color-media="(prefers-color-scheme: light)" data-md-color-scheme="default" data-md-color-primary="black" data-md-color-accent="blue" aria-hidden="true" type="radio" name="__palette" id="__palette_1">
</form>
<script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
<label class="md-header__button md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
</label>
<div class="md-search" data-md-component="search" role="dialog">
<label class="md-search__overlay" for="__search"></label>
<div class="md-search__inner" role="search">
<form class="md-search__form" name="search">
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
<label class="md-search__icon md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
</label>
<nav class="md-search__options" aria-label="Search">
<button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
</button>
</nav>
<div class="md-search__suggest" data-md-component="search-suggest"></div>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
<div class="md-search-result" data-md-component="search-result">
<div class="md-search-result__meta">
Initializing search
</div>
<ol class="md-search-result__list" role="presentation"></ol>
</div>
</div>
</div>
</div>
</div>
</nav>
</header>
<div class="md-container" data-md-component="container">
<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
<div class="md-grid">
<ul class="md-tabs__list">
<li class="md-tabs__item">
<a href="../.." class="md-tabs__link">
Home
</a>
</li>
<li class="md-tabs__item">
<a href="../../CHANGELOG/" class="md-tabs__link">
Changelog
</a>
</li>
<li class="md-tabs__item">
<a href="../../RESUME/" class="md-tabs__link">
Resume
</a>
</li>
<li class="md-tabs__item">
<a href="../../ct-inventory/" class="md-tabs__link">
CT inventory
</a>
</li>
<li class="md-tabs__item">
<a href="../../infra/proxmox-state/" class="md-tabs__link">
Infra
</a>
</li>
<li class="md-tabs__item md-tabs__item--active">
<a href="../homelab-architecture/" class="md-tabs__link">
Services
</a>
</li>
<li class="md-tabs__item">
<a href="../../security/data-leak-audit-comparison/" class="md-tabs__link">
Security & audits
</a>
</li>
<li class="md-tabs__item">
<a href="../../ideas/stack-ideas/" class="md-tabs__link">
Ideas
</a>
</li>
<li class="md-tabs__item">
<a href="../../history/traefik-migration/" class="md-tabs__link">
History
</a>
</li>
</ul>
</div>
</nav>
<main class="md-main" data-md-component="main">
<div class="md-main__inner md-grid">
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary md-nav--lifted" aria-label="Navigation" data-md-level="0">
<label class="md-nav__title" for="__drawer">
<a href="../.." title="nuclide.systems docs" class="md-nav__button md-logo" aria-label="nuclide.systems docs" data-md-component="logo">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
</a>
nuclide.systems docs
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../.." class="md-nav__link">
<span class="md-ellipsis">
Home
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../CHANGELOG/" class="md-nav__link">
<span class="md-ellipsis">
Changelog
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../RESUME/" class="md-nav__link">
<span class="md-ellipsis">
Resume
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../ct-inventory/" class="md-nav__link">
<span class="md-ellipsis">
CT inventory
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_5" >
<label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="0">
<span class="md-ellipsis">
Infra
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_5">
<span class="md-nav__icon md-icon"></span>
Infra
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../infra/proxmox-state/" class="md-nav__link">
<span class="md-ellipsis">
Proxmox state
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../infra/portmap/" class="md-nav__link">
<span class="md-ellipsis">
Port map
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../infra/storage/" class="md-nav__link">
<span class="md-ellipsis">
Storage
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../infra/volumes/" class="md-nav__link">
<span class="md-ellipsis">
Volumes
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../infra/docker-networks/" class="md-nav__link">
<span class="md-ellipsis">
Docker networks
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../infra/connection-hosts/" class="md-nav__link">
<span class="md-ellipsis">
Connection hosts
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../infra/proxmox-memory-audit/" class="md-nav__link">
<span class="md-ellipsis">
Memory audit
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6" checked>
<label class="md-nav__link" for="__nav_6" id="__nav_6_label" tabindex="">
<span class="md-ellipsis">
Services
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_6_label" aria-expanded="true">
<label class="md-nav__title" for="__nav_6">
<span class="md-nav__icon md-icon"></span>
Services
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../homelab-architecture/" class="md-nav__link">
<span class="md-ellipsis">
Homelab architecture
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../dev-environment/" class="md-nav__link">
<span class="md-ellipsis">
Dev environment (Coder + Gitea)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../mcp-gateway/" class="md-nav__link">
<span class="md-ellipsis">
MCP gateway
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../mcp-servers/" class="md-nav__link">
<span class="md-ellipsis">
MCP servers
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../pocket-id/" class="md-nav__link">
<span class="md-ellipsis">
Pocket-ID (OIDC)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../backrest/" class="md-nav__link">
<span class="md-ellipsis">
Backrest (backups)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../databases/" class="md-nav__link">
<span class="md-ellipsis">
Databases
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../secrets-manager/" class="md-nav__link">
<span class="md-ellipsis">
Secrets manager
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../comfyui/" class="md-nav__link">
<span class="md-ellipsis">
ComfyUI
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../zoraxy/" class="md-nav__link">
<span class="md-ellipsis">
Zoraxy
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../adguard-dns/" class="md-nav__link">
<span class="md-ellipsis">
AdGuard DNS
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../cloud-gpu/" class="md-nav__link">
<span class="md-ellipsis">
Cloud GPU
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../arcane/" class="md-nav__link">
<span class="md-ellipsis">
Arcane
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--active">
<input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
<label class="md-nav__link md-nav__link--active" for="__toc">
<span class="md-ellipsis">
Doc ingestion pipeline
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<a href="./" class="md-nav__link md-nav__link--active">
<span class="md-ellipsis">
Doc ingestion pipeline
</span>
</a>
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#architecture" class="md-nav__link">
<span class="md-ellipsis">
Architecture
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#deployed-components-as-of-2026-05-26" class="md-nav__link">
<span class="md-ellipsis">
Deployed components (as of 2026-05-26)
</span>
</a>
<nav class="md-nav" aria-label="Deployed components (as of 2026-05-26)">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#embedding-stack" class="md-nav__link">
<span class="md-ellipsis">
Embedding stack
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#owui-rag-config-env-driven" class="md-nav__link">
<span class="md-ellipsis">
OWUI RAG config (env-driven)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#bifrost-embedding-models-available-for-external-services-upgrade" class="md-nav__link">
<span class="md-ellipsis">
Bifrost embedding models (available for external services / upgrade)
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#converter-comparison" class="md-nav__link">
<span class="md-ellipsis">
Converter comparison
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#sharing-qdrant-with-other-services" class="md-nav__link">
<span class="md-ellipsis">
Sharing Qdrant with other services
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#bifrost-semantic-caching" class="md-nav__link">
<span class="md-ellipsis">
Bifrost semantic caching
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#intel-arc-gpu-passthrough-enabled" class="md-nav__link">
<span class="md-ellipsis">
Intel Arc GPU passthrough (enabled)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#pending" class="md-nav__link">
<span class="md-ellipsis">
Pending
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../llm-benchmark/" class="md-nav__link">
<span class="md-ellipsis">
LLM benchmark
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_7" >
<label class="md-nav__link" for="__nav_7" id="__nav_7_label" tabindex="0">
<span class="md-ellipsis">
Security & audits
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_7_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_7">
<span class="md-nav__icon md-icon"></span>
Security & audits
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../security/data-leak-audit-comparison/" class="md-nav__link">
<span class="md-ellipsis">
Comparison
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../security/data-leak-audit-2026-05-20-tr004-cloud-sandbox/" class="md-nav__link">
<span class="md-ellipsis">
2026-05-20 · cloud-sandbox breach
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../security/data-leak-audit-2026-05-21-tr004-artifacts/" class="md-nav__link">
<span class="md-ellipsis">
2026-05-21 · artifacts (clean)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../security/audit-claude-code-meta/" class="md-nav__link">
<span class="md-ellipsis">
Self-audit (Claude Code)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../security/transcripts/audit-chat-2026-05-21/" class="md-nav__link">
<span class="md-ellipsis">
Transcript (audit session)
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_8" >
<label class="md-nav__link" for="__nav_8" id="__nav_8_label" tabindex="0">
<span class="md-ellipsis">
Ideas
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_8">
<span class="md-nav__icon md-icon"></span>
Ideas
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../ideas/stack-ideas/" class="md-nav__link">
<span class="md-ellipsis">
Stack ideas
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_9" >
<label class="md-nav__link" for="__nav_9" id="__nav_9_label" tabindex="0">
<span class="md-ellipsis">
History
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_9_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_9">
<span class="md-nav__icon md-icon"></span>
History
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../history/traefik-migration/" class="md-nav__link">
<span class="md-ellipsis">
Traefik (abandoned 2026-05-16)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../history/traefik-migration-docker-labels/" class="md-nav__link">
<span class="md-ellipsis">
Traefik labels (abandoned)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../history/mcp-gateway-requirements/" class="md-nav__link">
<span class="md-ellipsis">
MCP gateway requirements (superseded)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../history/scrubbing-list-2026-05-17/" class="md-nav__link">
<span class="md-ellipsis">
Scrubbing list (2026-05-17)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../../history/case-study/" class="md-nav__link">
<span class="md-ellipsis">
Case study
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#architecture" class="md-nav__link">
<span class="md-ellipsis">
Architecture
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#deployed-components-as-of-2026-05-26" class="md-nav__link">
<span class="md-ellipsis">
Deployed components (as of 2026-05-26)
</span>
</a>
<nav class="md-nav" aria-label="Deployed components (as of 2026-05-26)">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#embedding-stack" class="md-nav__link">
<span class="md-ellipsis">
Embedding stack
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#owui-rag-config-env-driven" class="md-nav__link">
<span class="md-ellipsis">
OWUI RAG config (env-driven)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#bifrost-embedding-models-available-for-external-services-upgrade" class="md-nav__link">
<span class="md-ellipsis">
Bifrost embedding models (available for external services / upgrade)
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#converter-comparison" class="md-nav__link">
<span class="md-ellipsis">
Converter comparison
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#sharing-qdrant-with-other-services" class="md-nav__link">
<span class="md-ellipsis">
Sharing Qdrant with other services
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#bifrost-semantic-caching" class="md-nav__link">
<span class="md-ellipsis">
Bifrost semantic caching
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#intel-arc-gpu-passthrough-enabled" class="md-nav__link">
<span class="md-ellipsis">
Intel Arc GPU passthrough (enabled)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#pending" class="md-nav__link">
<span class="md-ellipsis">
Pending
</span>
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content" data-md-component="content">
<article class="md-content__inner md-typeset">
<h1 id="document-ingestion-pipeline">Document Ingestion Pipeline<a class="headerlink" href="#document-ingestion-pipeline" title="Permanent link">&para;</a></h1>
<p>Ingest PDFs, Word, PPTX, XLSX from Nextcloud/Paperless into Open WebUI's
knowledge base and make them searchable via MCP.</p>
<h2 id="architecture">Architecture<a class="headerlink" href="#architecture" title="Permanent link">&para;</a></h2>
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a>Nextcloud folder / Paperless webhook
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a> → n8n trigger (CT 104)
<a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a> → Docling MCP (port 18005) — PDF/DOCX/PPTX/XLSX → Markdown + structure
<a id="__codelineno-0-4" name="__codelineno-0-4" href="#__codelineno-0-4"></a> → TEI /v1/embeddings — multilingual-e5-base (local, 768d)
<a id="__codelineno-0-5" name="__codelineno-0-5" href="#__codelineno-0-5"></a> → Qdrant (shared vector store, http://qdrant:6333)
<a id="__codelineno-0-6" name="__codelineno-0-6" href="#__codelineno-0-6"></a> → Open WebUI knowledge base API ← searchable in chat
</code></pre></div>
<p><strong>Paperless shortcut</strong>: Paperless-ngx already OCRs documents. Its full-text content is
available at <code>/api/documents/?added__gt=&lt;last_run&gt;</code>. An n8n workflow can re-embed
directly from Paperless's REST API without re-running Docling for already-OCR'd PDFs.</p>
<h2 id="deployed-components-as-of-2026-05-26">Deployed components (as of 2026-05-26)<a class="headerlink" href="#deployed-components-as-of-2026-05-26" title="Permanent link">&para;</a></h2>
<table>
<thead>
<tr>
<th>Component</th>
<th>Location</th>
<th>Endpoint</th>
</tr>
</thead>
<tbody>
<tr>
<td><strong>Qdrant</strong></td>
<td>CT 104, <code>ai-internal</code> net</td>
<td><code>http://qdrant:6333</code> (REST), <code>qdrant:6334</code> (gRPC)</td>
</tr>
<tr>
<td><strong>nomic</strong></td>
<td>CT 104, <code>ai-internal</code> net</td>
<td><code>http://nomic:80</code> — text+vision 768d</td>
</tr>
<tr>
<td><strong>TEI</strong> (Text Embeddings Inference)</td>
<td>CT 104, <code>ai-internal</code> net</td>
<td><code>http://tei:80</code> — text-only 768d (standby)</td>
</tr>
<tr>
<td><strong>Open WebUI</strong></td>
<td>CT 104, port 14002</td>
<td>Uses Qdrant + nomic natively</td>
</tr>
<tr>
<td><strong>Bifrost</strong></td>
<td>CT 104, port 14003</td>
<td>Semantic cache → Qdrant gRPC, mistral-embed 1024d</td>
</tr>
<tr>
<td><strong>Docling MCP</strong></td>
<td>CT 104, port 18005</td>
<td>MCP server in gateway</td>
</tr>
</tbody>
</table>
<h3 id="embedding-stack">Embedding stack<a class="headerlink" href="#embedding-stack" title="Permanent link">&para;</a></h3>
<table>
<thead>
<tr>
<th>Service</th>
<th>Model</th>
<th>Dimensions</th>
<th>Use</th>
</tr>
</thead>
<tbody>
<tr>
<td><strong>nomic</strong></td>
<td><code>nomic-ai/nomic-embed-text-v1.5</code> + <code>nomic-embed-vision-v1.5</code></td>
<td>768d</td>
<td>OWUI RAG, ingest pipeline</td>
</tr>
<tr>
<td><strong>TEI</strong></td>
<td><code>intfloat/multilingual-e5-base</code></td>
<td>768d</td>
<td>Standby; same vector space as nomic text</td>
</tr>
<tr>
<td><strong>Bifrost cache</strong></td>
<td><code>mistral/mistral-embed</code> via Bifrost</td>
<td>1024d</td>
<td>Semantic cache only (separate Qdrant collection)</td>
</tr>
</tbody>
</table>
<p>Both nomic models share a 768d embedding space — text and image queries work on the same <code>documents</code> Qdrant collection.</p>
<h3 id="owui-rag-config-env-driven">OWUI RAG config (env-driven)<a class="headerlink" href="#owui-rag-config-env-driven" title="Permanent link">&para;</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a>VECTOR_DB=qdrant
<a id="__codelineno-1-2" name="__codelineno-1-2" href="#__codelineno-1-2"></a>QDRANT_URI=http://qdrant:6333
<a id="__codelineno-1-3" name="__codelineno-1-3" href="#__codelineno-1-3"></a>RAG_EMBEDDING_ENGINE=openai
<a id="__codelineno-1-4" name="__codelineno-1-4" href="#__codelineno-1-4"></a>RAG_OPENAI_API_BASE_URL=http://nomic:80
<a id="__codelineno-1-5" name="__codelineno-1-5" href="#__codelineno-1-5"></a>RAG_OPENAI_API_KEY=none
<a id="__codelineno-1-6" name="__codelineno-1-6" href="#__codelineno-1-6"></a>RAG_EMBEDDING_MODEL=nomic-ai/nomic-embed-text-v1.5
<a id="__codelineno-1-7" name="__codelineno-1-7" href="#__codelineno-1-7"></a>CONTENT_EXTRACTION_ENGINE=docling
<a id="__codelineno-1-8" name="__codelineno-1-8" href="#__codelineno-1-8"></a>CHUNK_SIZE=1200
<a id="__codelineno-1-9" name="__codelineno-1-9" href="#__codelineno-1-9"></a>CHUNK_OVERLAP=150
<a id="__codelineno-1-10" name="__codelineno-1-10" href="#__codelineno-1-10"></a>ENABLE_RAG_HYBRID_SEARCH=true
</code></pre></div>
<p>To switch to Bifrost embeddings (1024d, better quality — requires re-indexing <code>documents</code> collection):
<div class="highlight"><pre><span></span><code><a id="__codelineno-2-1" name="__codelineno-2-1" href="#__codelineno-2-1"></a>RAG_OPENAI_API_BASE_URL=http://bifrost:8080/v1
<a id="__codelineno-2-2" name="__codelineno-2-2" href="#__codelineno-2-2"></a>RAG_EMBEDDING_MODEL=mistral/mistral-embed
</code></pre></div></p>
<h3 id="bifrost-embedding-models-available-for-external-services-upgrade">Bifrost embedding models (available for external services / upgrade)<a class="headerlink" href="#bifrost-embedding-models-available-for-external-services-upgrade" title="Permanent link">&para;</a></h3>
<p>Three embedding models tested and working via <code>http://bifrost:8080/v1/embeddings</code>:
- <code>mistral/mistral-embed</code> (1024d) — ✓ production-ready
- <code>mistral/codestral-embed</code> (1024d) — ✓
- <code>gemini/gemini-embedding-001</code> (768d/1536d) — ✓</p>
<h2 id="converter-comparison">Converter comparison<a class="headerlink" href="#converter-comparison" title="Permanent link">&para;</a></h2>
<table>
<thead>
<tr>
<th>Tool</th>
<th>Image</th>
<th>Formats</th>
<th>Notes</th>
</tr>
</thead>
<tbody>
<tr>
<td><strong>Docling</strong> (deployed)</td>
<td>MCP on 18005</td>
<td>PDF, DOCX, PPTX, XLSX, HTML</td>
<td>Best for structured Office/PDF with tables</td>
</tr>
<tr>
<td><strong>MinerU</strong></td>
<td><code>opendatalab/mineru</code></td>
<td>PDF (layout-aware, OCR)</td>
<td>Better for academic papers / scanned PDFs</td>
</tr>
<tr>
<td><strong>Markitdown</strong> (in gateway)</td>
<td>—</td>
<td>Office, PDF</td>
<td>Ad-hoc only; not suitable for batch</td>
</tr>
</tbody>
</table>
<p>Start with Docling — already deployed. Add MinerU if academic paper OCR quality is needed.</p>
<h2 id="sharing-qdrant-with-other-services">Sharing Qdrant with other services<a class="headerlink" href="#sharing-qdrant-with-other-services" title="Permanent link">&para;</a></h2>
<p>Qdrant is on <code>ai-internal</code> network — any service on that network can use it:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-3-1" name="__codelineno-3-1" href="#__codelineno-3-1"></a><span class="kn">from</span><span class="w"> </span><span class="nn">qdrant_client</span><span class="w"> </span><span class="kn">import</span> <span class="n">QdrantClient</span>
<a id="__codelineno-3-2" name="__codelineno-3-2" href="#__codelineno-3-2"></a><span class="n">client</span> <span class="o">=</span> <span class="n">QdrantClient</span><span class="p">(</span><span class="n">url</span><span class="o">=</span><span class="s2">&quot;http://qdrant:6333&quot;</span><span class="p">)</span>
</code></pre></div>
<p>n8n, MCP tools, and custom pipelines should use <code>http://nomic:80/v1/embeddings</code> for
consistent 768d vectors. Mixing models/dimensions in the same collection will fail.</p>
<h2 id="bifrost-semantic-caching">Bifrost semantic caching<a class="headerlink" href="#bifrost-semantic-caching" title="Permanent link">&para;</a></h2>
<p>Bifrost uses Qdrant (gRPC port 6334) as a semantic cache backend. Config lives in
<code>/opt/stacks/ai/bifrost/data/config.json</code>:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-4-1" name="__codelineno-4-1" href="#__codelineno-4-1"></a><span class="p">{</span>
<a id="__codelineno-4-2" name="__codelineno-4-2" href="#__codelineno-4-2"></a><span class="w"> </span><span class="nt">&quot;$schema&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;https://www.getbifrost.ai/schema&quot;</span><span class="p">,</span>
<a id="__codelineno-4-3" name="__codelineno-4-3" href="#__codelineno-4-3"></a><span class="w"> </span><span class="nt">&quot;vector_store&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="nt">&quot;enabled&quot;</span><span class="p">:</span><span class="w"> </span><span class="kc">true</span><span class="p">,</span><span class="w"> </span><span class="nt">&quot;type&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;qdrant&quot;</span><span class="p">,</span><span class="w"> </span><span class="nt">&quot;config&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="nt">&quot;host&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;qdrant&quot;</span><span class="p">,</span><span class="w"> </span><span class="nt">&quot;port&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">6334</span><span class="p">}},</span>
<a id="__codelineno-4-4" name="__codelineno-4-4" href="#__codelineno-4-4"></a><span class="w"> </span><span class="nt">&quot;plugins&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">[{</span>
<a id="__codelineno-4-5" name="__codelineno-4-5" href="#__codelineno-4-5"></a><span class="w"> </span><span class="nt">&quot;enabled&quot;</span><span class="p">:</span><span class="w"> </span><span class="kc">true</span><span class="p">,</span><span class="w"> </span><span class="nt">&quot;name&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;semantic_cache&quot;</span><span class="p">,</span>
<a id="__codelineno-4-6" name="__codelineno-4-6" href="#__codelineno-4-6"></a><span class="w"> </span><span class="nt">&quot;config&quot;</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
<a id="__codelineno-4-7" name="__codelineno-4-7" href="#__codelineno-4-7"></a><span class="w"> </span><span class="nt">&quot;provider&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;mistral&quot;</span><span class="p">,</span><span class="w"> </span><span class="nt">&quot;embedding_model&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;mistral-embed&quot;</span><span class="p">,</span><span class="w"> </span><span class="nt">&quot;dimension&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">1024</span><span class="p">,</span>
<a id="__codelineno-4-8" name="__codelineno-4-8" href="#__codelineno-4-8"></a><span class="w"> </span><span class="nt">&quot;ttl&quot;</span><span class="p">:</span><span class="w"> </span><span class="s2">&quot;10m&quot;</span><span class="p">,</span><span class="w"> </span><span class="nt">&quot;threshold&quot;</span><span class="p">:</span><span class="w"> </span><span class="mf">0.85</span><span class="p">,</span><span class="w"> </span><span class="nt">&quot;conversation_history_threshold&quot;</span><span class="p">:</span><span class="w"> </span><span class="mi">3</span><span class="p">,</span><span class="w"> </span><span class="nt">&quot;exclude_system_prompt&quot;</span><span class="p">:</span><span class="w"> </span><span class="kc">true</span>
<a id="__codelineno-4-9" name="__codelineno-4-9" href="#__codelineno-4-9"></a><span class="w"> </span><span class="p">}</span>
<a id="__codelineno-4-10" name="__codelineno-4-10" href="#__codelineno-4-10"></a><span class="w"> </span><span class="p">}]</span>
<a id="__codelineno-4-11" name="__codelineno-4-11" href="#__codelineno-4-11"></a><span class="p">}</span>
</code></pre></div>
<p>The semantic cache uses a separate Qdrant collection (auto-created) at 1024d — no collision with the <code>documents</code> collection at 768d. TTL: 10 min, similarity threshold: 0.85.</p>
<h2 id="intel-arc-gpu-passthrough-enabled">Intel Arc GPU passthrough (enabled)<a class="headerlink" href="#intel-arc-gpu-passthrough-enabled" title="Permanent link">&para;</a></h2>
<p>CT 104 Intel Core Ultra 7 155H iGPU is passed through via PVE <code>dev3</code>/<code>dev4</code> entries
(<code>/dev/dri/renderD128</code> and <code>/dev/dri/card1</code>). No TEI Intel image exists currently — passthrough is available for future inference acceleration.</p>
<h2 id="pending">Pending<a class="headerlink" href="#pending" title="Permanent link">&para;</a></h2>
<ul>
<li>n8n workflow: Nextcloud/Paperless → Docling → nomic → Qdrant</li>
<li>MCP knowledge-search tool pointing at Qdrant (replaces LobeChat search intent)</li>
<li><code>ingest.py</code> filesystem scan script (see <a href="../ingest-pipeline/">ingest-pipeline.md</a>)</li>
</ul>
</article>
</div>
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
</div>
<button type="button" class="md-top md-icon" data-md-component="top" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8z"/></svg>
Back to top
</button>
</main>
<footer class="md-footer">
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-copyright">
Made with
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
Material for MkDocs
</a>
</div>
</div>
</div>
</footer>
</div>
<div class="md-dialog" data-md-component="dialog">
<div class="md-dialog__inner md-typeset"></div>
</div>
<script id="__config" type="application/json">{"annotate": null, "base": "../..", "features": ["navigation.tabs", "navigation.sections", "navigation.expand", "navigation.top", "search.highlight", "search.suggest", "content.code.copy"], "search": "../../assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
<script src="../../assets/javascripts/bundle.79ae519e.min.js"></script>
</body>
</html>
Reference in New Issue View Git Blame Copy Permalink