Files
docs/omniread/wiki/index.html
Vishesh 'ironeagle' Bangotra aedd2061e1 feat: collect omniread wiki; register wiki kind; refresh lib and mcp artifacts
Adds the omniread wiki to the hub alongside lib and mcp and completes the
library nav with csv/xlsx groups, picking up regenerated MCP modules.
2026-09-16 19:59:00 +05:30

661 lines
20 KiB
HTML
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

<!doctype html>
<html lang="en" class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<link rel="next" href="01_overview/">
<link rel="icon" href="assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.6.23">
<title>OmniRead Documentation</title>
<link rel="stylesheet" href="assets/stylesheets/main.84d31ad4.min.css">
<link rel="stylesheet" href="assets/stylesheets/palette.06af60db.min.css">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CJetBrains+Mono:400,400i,700,700i&display=fallback">
<style>:root{--md-text-font:"Roboto";--md-code-font:"JetBrains Mono"}</style>
<link rel="stylesheet" href="https://unpkg.com/dracula-prism/dist/css/dracula-prism.css">
<script>__md_scope=new URL(".",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
</head>
<body dir="ltr" data-md-color-scheme="slate" data-md-color-primary="blue-grey" data-md-color-accent="teal">
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" for="__drawer"></label>
<div data-md-component="skip">
<a href="#omniread-format-agnostic-content-acquisition-and-parsing" class="md-skip">
Skip to content
</a>
</div>
<div data-md-component="announce">
</div>
<header class="md-header" data-md-component="header">
<nav class="md-header__inner md-grid" aria-label="Header">
<a href="." title="OmniRead Documentation" class="md-header__button md-logo" aria-label="OmniRead Documentation" data-md-component="logo">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="m19 2-5 4.5v11l5-4.5zM6.5 5C4.55 5 2.45 5.4 1 6.5v14.66c0 .25.25.5.5.5.1 0 .15-.07.25-.07 1.35-.65 3.3-1.09 4.75-1.09 1.95 0 4.05.4 5.5 1.5 1.35-.85 3.8-1.5 5.5-1.5 1.65 0 3.35.31 4.75 1.06.1.05.15.03.25.03.25 0 .5-.25.5-.5V6.5c-.6-.45-1.25-.75-2-1V19c-1.1-.35-2.3-.5-3.5-.5-1.7 0-4.15.65-5.5 1.5V6.5C10.55 5.4 8.45 5 6.5 5"/></svg>
</a>
<label class="md-header__button md-icon" for="__drawer">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
</label>
<div class="md-header__title" data-md-component="header-title">
<div class="md-header__ellipsis">
<div class="md-header__topic">
<span class="md-ellipsis">
OmniRead Documentation
</span>
</div>
<div class="md-header__topic" data-md-component="header-topic">
<span class="md-ellipsis">
Home
</span>
</div>
</div>
</div>
<label class="md-header__button md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
</label>
<div class="md-search" data-md-component="search" role="dialog">
<label class="md-search__overlay" for="__search"></label>
<div class="md-search__inner" role="search">
<form class="md-search__form" name="search">
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
<label class="md-search__icon md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
</label>
<nav class="md-search__options" aria-label="Search">
<a href="javascript:void(0)" class="md-search__icon md-icon" title="Share" aria-label="Share" data-clipboard data-clipboard-text="" data-md-component="search-share" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M18 16.08c-.76 0-1.44.3-1.96.77L8.91 12.7c.05-.23.09-.46.09-.7s-.04-.47-.09-.7l7.05-4.11c.54.5 1.25.81 2.04.81a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3c0 .24.04.47.09.7L8.04 9.81C7.5 9.31 6.79 9 6 9a3 3 0 0 0-3 3 3 3 0 0 0 3 3c.79 0 1.5-.31 2.04-.81l7.12 4.15c-.05.21-.08.43-.08.66 0 1.61 1.31 2.91 2.92 2.91s2.92-1.3 2.92-2.91A2.92 2.92 0 0 0 18 16.08"/></svg>
</a>
<button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
</button>
</nav>
<div class="md-search__suggest" data-md-component="search-suggest"></div>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
<div class="md-search-result" data-md-component="search-result">
<div class="md-search-result__meta">
Initializing search
</div>
<ol class="md-search-result__list" role="presentation"></ol>
</div>
</div>
</div>
</div>
</div>
</nav>
</header>
<div class="md-container" data-md-component="container">
<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
<div class="md-grid">
<ul class="md-tabs__list">
<li class="md-tabs__item md-tabs__item--active">
<a href="." class="md-tabs__link">
Home
</a>
</li>
<li class="md-tabs__item">
<a href="01_overview/" class="md-tabs__link">
Overview
</a>
</li>
<li class="md-tabs__item">
<a href="02_how_to_use/" class="md-tabs__link">
How to Use
</a>
</li>
<li class="md-tabs__item">
<a href="03_extending/" class="md-tabs__link">
Extending OmniRead
</a>
</li>
<li class="md-tabs__item">
<a href="04_development/" class="md-tabs__link">
Development
</a>
</li>
</ul>
</div>
</nav>
<main class="md-main" data-md-component="main">
<div class="md-main__inner md-grid">
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary md-nav--lifted md-nav--integrated" aria-label="Navigation" data-md-level="0">
<label class="md-nav__title" for="__drawer">
<a href="." title="OmniRead Documentation" class="md-nav__button md-logo" aria-label="OmniRead Documentation" data-md-component="logo">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="m19 2-5 4.5v11l5-4.5zM6.5 5C4.55 5 2.45 5.4 1 6.5v14.66c0 .25.25.5.5.5.1 0 .15-.07.25-.07 1.35-.65 3.3-1.09 4.75-1.09 1.95 0 4.05.4 5.5 1.5 1.35-.85 3.8-1.5 5.5-1.5 1.65 0 3.35.31 4.75 1.06.1.05.15.03.25.03.25 0 .5-.25.5-.5V6.5c-.6-.45-1.25-.75-2-1V19c-1.1-.35-2.3-.5-3.5-.5-1.7 0-4.15.65-5.5 1.5V6.5C10.55 5.4 8.45 5 6.5 5"/></svg>
</a>
OmniRead Documentation
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item md-nav__item--active">
<input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
<label class="md-nav__link md-nav__link--active" for="__toc">
<span class="md-ellipsis">
Home
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<a href="." class="md-nav__link md-nav__link--active">
<span class="md-ellipsis">
Home
</span>
</a>
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#key-features" class="md-nav__link">
<span class="md-ellipsis">
🚀 Key Features
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#quick-start" class="md-nav__link">
<span class="md-ellipsis">
⚡ Quick Start
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#documentation-structure" class="md-nav__link">
<span class="md-ellipsis">
📁 Documentation Structure
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#related-resources" class="md-nav__link">
<span class="md-ellipsis">
🔗 Related Resources
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="01_overview/" class="md-nav__link">
<span class="md-ellipsis">
Overview
</span>
</a>
</li>
<li class="md-nav__item">
<a href="02_how_to_use/" class="md-nav__link">
<span class="md-ellipsis">
How to Use
</span>
</a>
</li>
<li class="md-nav__item">
<a href="03_extending/" class="md-nav__link">
<span class="md-ellipsis">
Extending OmniRead
</span>
</a>
</li>
<li class="md-nav__item">
<a href="04_development/" class="md-nav__link">
<span class="md-ellipsis">
Development
</span>
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content" data-md-component="content">
<article class="md-content__inner md-typeset">
<h1 id="omniread-format-agnostic-content-acquisition-and-parsing">📄 OmniRead — Format-Agnostic Content Acquisition and Parsing</h1>
<p><code>OmniRead</code> is a composable content ingestion framework with pluggable scrapers
and parsers for HTML, PDF, CSV, and XLSX. It cleanly separates fetching
(I/O) from interpretation (parsing) through a canonical <code>Content</code> exchange
model, giving each format its own scraper and parser while sharing one
uniform pipeline.</p>
<blockquote>
<p><strong>Doc model:</strong> this wiki is written for humans — howto guides, examples,
and extension recipes. The authoritative API contracts live in the code
(GSDFC docstrings) and the machinereadable bundle under <code>docs/mcp/</code>.</p>
</blockquote>
<hr />
<h2 id="key-features">🚀 Key Features</h2>
<ul>
<li>🧱 <strong>Content/Scraper/Parser architecture</strong> — fetch and interpret in
independent, testable layers</li>
<li>📄 <strong>Canonical <code>Content</code> model</strong> — raw bytes + metadata passed between all
layers via one dataclass</li>
<li>🌐 <strong>HTML</strong> — HTTP-based scraper with httpx; DOM parsing via BeautifulSoup</li>
<li>📕 <strong>PDF</strong> — client + scraper for raw PDF bytes; user-defined parsers</li>
<li>📊 <strong>CSV</strong> — standard-library CSV parser over <code>rows()</code> iterator</li>
<li>📑 <strong>XLSX</strong> — openpyxl-backed spreadsheet parser with sheet selection,
empty-row control, and ISO date conversion</li>
</ul>
<hr />
<h2 id="quick-start">⚡ Quick Start</h2>
<div class="language-python highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1">1</a></span>
<span class="normal"><a href="#__codelineno-0-2">2</a></span>
<span class="normal"><a href="#__codelineno-0-3">3</a></span>
<span class="normal"><a href="#__codelineno-0-4">4</a></span>
<span class="normal"><a href="#__codelineno-0-5">5</a></span>
<span class="normal"><a href="#__codelineno-0-6">6</a></span>
<span class="normal"><a href="#__codelineno-0-7">7</a></span>
<span class="normal"><a href="#__codelineno-0-8">8</a></span>
<span class="normal"><a href="#__codelineno-0-9">9</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-1"><a id="__codelineno-0-1" name="__codelineno-0-1"></a><span class="kn">from</span><span class="w"> </span><span class="nn">omniread</span><span class="w"> </span><span class="kn">import</span> <span class="n">HTMLScraper</span><span class="p">,</span> <span class="n">HTMLParser</span>
</span><span id="__span-0-2"><a id="__codelineno-0-2" name="__codelineno-0-2"></a>
</span><span id="__span-0-3"><a id="__codelineno-0-3" name="__codelineno-0-3"></a><span class="k">class</span><span class="w"> </span><span class="nc">TitleParser</span><span class="p">(</span><span class="n">HTMLParser</span><span class="p">[</span><span class="nb">str</span><span class="p">]):</span>
</span><span id="__span-0-4"><a id="__codelineno-0-4" name="__codelineno-0-4"></a> <span class="k">def</span><span class="w"> </span><span class="nf">parse</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
</span><span id="__span-0-5"><a id="__codelineno-0-5" name="__codelineno-0-5"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_soup</span><span class="o">.</span><span class="n">title</span><span class="o">.</span><span class="n">string</span>
</span><span id="__span-0-6"><a id="__codelineno-0-6" name="__codelineno-0-6"></a>
</span><span id="__span-0-7"><a id="__codelineno-0-7" name="__codelineno-0-7"></a><span class="n">content</span> <span class="o">=</span> <span class="n">HTMLScraper</span><span class="p">()</span><span class="o">.</span><span class="n">fetch</span><span class="p">(</span><span class="s2">&quot;https://example.com&quot;</span><span class="p">)</span>
</span><span id="__span-0-8"><a id="__codelineno-0-8" name="__codelineno-0-8"></a><span class="n">title</span> <span class="o">=</span> <span class="n">TitleParser</span><span class="p">(</span><span class="n">content</span><span class="p">)</span><span class="o">.</span><span class="n">parse</span><span class="p">()</span>
</span><span id="__span-0-9"><a id="__codelineno-0-9" name="__codelineno-0-9"></a><span class="k">assert</span> <span class="n">title</span> <span class="o">==</span> <span class="s2">&quot;Example Domain&quot;</span>
</span></code></pre></div></td></tr></table></div>
<hr />
<h2 id="documentation-structure">📁 Documentation Structure</h2>
<table>
<thead>
<tr>
<th>Section</th>
<th>What you'll find</th>
</tr>
</thead>
<tbody>
<tr>
<td><a href="01_overview/">Overview</a></td>
<td>The three-layer architecture and the <code>Content</code> model</td>
</tr>
<tr>
<td><a href="02_how_to_use/">How to Use</a></td>
<td>Ready-to-run examples for HTML, PDF, CSV, and XLSX</td>
</tr>
<tr>
<td><a href="03_extending/">Extending OmniRead</a></td>
<td>Subclassing scrapers, parsers, and clients</td>
</tr>
<tr>
<td><a href="04_development/">Development</a></td>
<td>Setup, tests, and regenerating docs</td>
</tr>
</tbody>
</table>
<hr />
<h2 id="related-resources">🔗 Related Resources</h2>
<ul>
<li><strong>Source Code:</strong> <a href="https://git.aetoskia.com/aetos/omniread">Gitea Repository</a></li>
<li><strong>Internal PyPI:</strong> <a href="https://pip.aetoskia.com/simple/omniread">pip.aetoskia.com/simple/omniread</a></li>
<li><strong>CI:</strong> Builds and publishes tagged releases, gated on black / ruff / mypy / pytest.</li>
</ul>
<hr />
<p>© Aetoskia Internal — <code>omniread</code> 0.0.1</p>
</article>
</div>
<script>var tabs=__md_get("__tabs");if(Array.isArray(tabs))e:for(var set of document.querySelectorAll(".tabbed-set")){var labels=set.querySelector(".tabbed-labels");for(var tab of tabs)for(var label of labels.getElementsByTagName("label"))if(label.innerText.trim()===tab){var input=document.getElementById(label.htmlFor);input.checked=!0;continue e}}</script>
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
</div>
<button type="button" class="md-top md-icon" data-md-component="top" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8z"/></svg>
Back to top
</button>
</main>
<footer class="md-footer">
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-copyright">
Made with
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
Material for MkDocs
</a>
</div>
</div>
</div>
</footer>
</div>
<div class="md-dialog" data-md-component="dialog">
<div class="md-dialog__inner md-typeset"></div>
</div>
<script id="__config" type="application/json">{"base": ".", "features": ["navigation.sections", "navigation.expand", "navigation.top", "navigation.instant", "navigation.tracking", "navigation.indexes", "content.code.copy", "content.code.annotate", "content.tabs.link", "content.action.edit", "search.highlight", "search.share", "search.suggest", "navigation.tabs", "toc.integrate", "header.autohide", "announce.dismiss", "footer.social", "content.code.select", "content.code.line_numbers", "content.tooltips"], "search": "assets/javascripts/workers/search.973d3a69.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
<script src="assets/javascripts/bundle.f55a23d4.min.js"></script>
</body>
</html>