aetos/docs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Build: 0.1.8
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source
This commit is contained in:
Vishesh 'ironeagle' Bangotra
2026-03-08 01:01:39 +05:30
parent 8701bf92ac
commit 9191de9dff
200 changed files with 130793 additions and 18277 deletions
Download Patch File Download Diff File Expand all files Collapse all files

450
libs/omniread/site/core/scraper/index.html
View File

@@ -86,7 +86,9 @@
<header class="md-header" data-md-component="header">
<header class="md-header md-header--shadow" data-md-component="header">
<nav class="md-header__inner md-grid" aria-label="Header">
<a href="../.." title="omniread" class="md-header__button md-logo" aria-label="omniread" data-md-component="logo">
@@ -151,12 +153,19 @@
</label>
<nav class="md-search__options" aria-label="Search">
<a href="javascript:void(0)" class="md-search__icon md-icon" title="Share" aria-label="Share" data-clipboard data-clipboard-text="" data-md-component="search-share" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M18 16.08c-.76 0-1.44.3-1.96.77L8.91 12.7c.05-.23.09-.46.09-.7s-.04-.47-.09-.7l7.05-4.11c.54.5 1.25.81 2.04.81a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3c0 .24.04.47.09.7L8.04 9.81C7.5 9.31 6.79 9 6 9a3 3 0 0 0-3 3 3 3 0 0 0 3 3c.79 0 1.5-.31 2.04-.81l7.12 4.15c-.05.21-.08.43-.08.66 0 1.61 1.31 2.91 2.92 2.91s2.92-1.3 2.92-2.91A2.92 2.92 0 0 0 18 16.08"/></svg>
</a>
<button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
</button>
</nav>
<div class="md-search__suggest" data-md-component="search-suggest"></div>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
@@ -182,96 +191,6 @@
<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
<div class="md-grid">
<ul class="md-tabs__list">
<li class="md-tabs__item">
<a href="../.." class="md-tabs__link">
Home
</a>
</li>
<li class="md-tabs__item md-tabs__item--active">
<a href="../" class="md-tabs__link">
Core API
</a>
</li>
<li class="md-tabs__item">
<a href="../../html/" class="md-tabs__link">
HTML Handling
</a>
</li>
<li class="md-tabs__item">
<a href="../../pdf/" class="md-tabs__link">
PDF Handling
</a>
</li>
</ul>
</div>
</nav>
<main class="md-main" data-md-component="main">
@@ -285,10 +204,8 @@
<nav class="md-nav md-nav--primary md-nav--lifted" aria-label="Navigation" data-md-level="0">
<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
<label class="md-nav__title" for="__drawer">
<a href="../.." title="omniread" class="md-nav__button md-logo" aria-label="omniread" data-md-component="logo">
@@ -332,6 +249,18 @@
@@ -339,7 +268,6 @@
<li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
@@ -349,8 +277,9 @@
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2" checked>
<label class="md-nav__link" for="__nav_2" id="__nav_2_label" tabindex="">
<div class="md-nav__link md-nav__container">
<a href="../" class="md-nav__link ">
<span class="md-ellipsis">
@@ -359,8 +288,14 @@
</span>
<span class="md-nav__icon md-icon"></span>
</label>
</a>
<label class="md-nav__link " for="__nav_2" id="__nav_2_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_2_label" aria-expanded="true">
<label class="md-nav__title" for="__nav_2">
@@ -370,27 +305,6 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../" class="md-nav__link">
<span class="md-ellipsis">
Core
</span>
</a>
</li>
@@ -505,6 +419,25 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#omniread.core.scraper--summary" class="md-nav__link">
<span class="md-ellipsis">
Summary
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#omniread.core.scraper-classes" class="md-nav__link">
<span class="md-ellipsis">
Classes
</span>
</a>
<nav class="md-nav" aria-label="Classes">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#omniread.core.scraper.BaseScraper" class="md-nav__link">
<span class="md-ellipsis">
BaseScraper
@@ -515,6 +448,16 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#omniread.core.scraper.BaseScraper-functions" class="md-nav__link">
<span class="md-ellipsis">
Functions
</span>
</a>
<nav class="md-nav" aria-label="Functions">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#omniread.core.scraper.BaseScraper.fetch" class="md-nav__link">
<span class="md-ellipsis">
fetch
@@ -531,6 +474,16 @@
</ul>
</nav>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
</ul>
@@ -557,13 +510,25 @@
<li class="md-nav__item md-nav__item--nested">
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
@@ -572,8 +537,9 @@
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_3" >
<label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="0">
<div class="md-nav__link md-nav__container">
<a href="../../html/" class="md-nav__link ">
<span class="md-ellipsis">
@@ -582,8 +548,14 @@
</span>
<span class="md-nav__icon md-icon"></span>
</label>
</a>
<label class="md-nav__link " for="__nav_3" id="__nav_3_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_3">
@@ -593,27 +565,6 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../html/" class="md-nav__link">
<span class="md-ellipsis">
Html
</span>
</a>
</li>
@@ -676,13 +627,27 @@
<li class="md-nav__item md-nav__item--nested">
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
@@ -691,8 +656,9 @@
<input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_4" >
<label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="0">
<div class="md-nav__link md-nav__container">
<a href="../../pdf/" class="md-nav__link ">
<span class="md-ellipsis">
@@ -701,8 +667,14 @@
</span>
<span class="md-nav__icon md-icon"></span>
</label>
</a>
<label class="md-nav__link " for="__nav_4" id="__nav_4_label" tabindex="">
<span class="md-nav__icon md-icon"></span>
</label>
</div>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="false">
<label class="md-nav__title" for="__nav_4">
@@ -712,27 +684,6 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../../pdf/" class="md-nav__link">
<span class="md-ellipsis">
Pdf
</span>
</a>
</li>
@@ -847,6 +798,25 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#omniread.core.scraper--summary" class="md-nav__link">
<span class="md-ellipsis">
Summary
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#omniread.core.scraper-classes" class="md-nav__link">
<span class="md-ellipsis">
Classes
</span>
</a>
<nav class="md-nav" aria-label="Classes">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#omniread.core.scraper.BaseScraper" class="md-nav__link">
<span class="md-ellipsis">
BaseScraper
@@ -857,6 +827,16 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#omniread.core.scraper.BaseScraper-functions" class="md-nav__link">
<span class="md-ellipsis">
Functions
</span>
</a>
<nav class="md-nav" aria-label="Functions">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#omniread.core.scraper.BaseScraper.fetch" class="md-nav__link">
<span class="md-ellipsis">
fetch
@@ -873,6 +853,16 @@
</ul>
</nav>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
</ul>
@@ -899,7 +889,7 @@
<h2 id="omniread.core.scraper" class="doc doc-heading">
<span class="doc doc-object-name doc-module-name">omniread.core.scraper</span>
<code class="doc-symbol doc-symbol-heading doc-symbol-module"></code> <span class="doc doc-object-name doc-module-name">omniread.core.scraper</span>
</h2>
@@ -907,6 +897,8 @@
<div class="doc doc-contents first">
<p>Abstract scraping contracts for OmniRead.</p>
<hr />
<h4 id="omniread.core.scraper--summary">Summary</h4>
<p>This module defines the <strong>format-agnostic scraper interface</strong> responsible for
acquiring raw content from external sources.</p>
<p>Scrapers are responsible for:
@@ -928,17 +920,17 @@ acquiring raw content from external sources.</p>
<h3 id="omniread.core.scraper-classes">Classes</h3>
<div class="doc doc-object doc-class">
<h3 id="omniread.core.scraper.BaseScraper" class="doc doc-heading">
<span class="doc doc-object-name doc-class-name">BaseScraper</span>
<h4 id="omniread.core.scraper.BaseScraper" class="doc doc-heading">
<code class="doc-symbol doc-symbol-heading doc-symbol-class"></code> <span class="doc doc-object-name doc-class-name">BaseScraper</span>
</h3>
</h4>
<div class="doc doc-contents ">
@@ -947,21 +939,24 @@ acquiring raw content from external sources.</p>
<p>Base interface for all scrapers.</p>
<p>A scraper is responsible ONLY for fetching raw content
(bytes) from a source. It must not interpret or parse it.</p>
<p>A scraper is a <strong>stateless acquisition component</strong> that retrieves raw
content from a source and returns it as a <code>Content</code> object.</p>
<p>Scrapers define <em>how content is obtained</em>, not <em>what the content means</em>.</p>
<p>Implementations may vary in:
- Transport mechanism (HTTP, filesystem, cloud storage)
- Authentication strategy
- Retry and backoff behavior</p>
<p>Implementations must not:
- Parse content
- Modify content semantics
- Couple scraping logic to a specific parser</p>
<details class="notes" open>
<summary>Notes</summary>
<p><strong>Responsibilities:</strong></p>
<div class="language-text highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">1</span>
<span class="normal">2</span>
<span class="normal">3</span>
<span class="normal">4</span></pre></div></td><td class="code"><div><pre><span></span><code>- A scraper is responsible ONLY for fetching raw content (bytes) from a source. It must not interpret or parse it
- A scraper is a stateless acquisition component that retrieves raw content from a source and returns it as a `Content` object
- Scrapers define how content is obtained, not what the content means
- Implementations may vary in transport mechanism, authentication strategy, retry and backoff behavior
</code></pre></div></td></tr></table></div>
<p><strong>Constraints:</strong></p>
<div class="language-text highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">1</span></pre></div></td><td class="code"><div><pre><span></span><code>- Implementations must not parse content, modify content semantics, or couple scraping logic to a specific parser
</code></pre></div></td></tr></table></div>
</details>
<div class="doc doc-children">
@@ -972,28 +967,26 @@ content from a source and returns it as a <code>Content</code> object.</p>
<h5 id="omniread.core.scraper.BaseScraper-functions">Functions</h5>
<div class="doc doc-object doc-function">
<h4 id="omniread.core.scraper.BaseScraper.fetch" class="doc doc-heading">
<span class="doc doc-object-name doc-function-name">fetch</span>
<h6 id="omniread.core.scraper.BaseScraper.fetch" class="doc doc-heading">
<code class="doc-symbol doc-symbol-heading doc-symbol-method"></code> <span class="doc doc-object-name doc-function-name">fetch</span>
<span class="doc doc-labels">
<small class="doc doc-label doc-label-abstractmethod"><code>abstractmethod</code></small>
</span>
</h4>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fetch</span><span class="p">(</span><span class="n">source</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">metadata</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Mapping</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Content</span>
</code></pre></div>
</h6>
<div class="language-python doc-signature highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-1">1</a></span></pre></div></td><td class="code"><div><pre><span></span><code><span id="__span-0-1"><a id="__codelineno-0-1" name="__codelineno-0-1"></a><span class="nf">fetch</span><span class="p">(</span><span class="n">source</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="o">*</span><span class="p">,</span> <span class="n">metadata</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Mapping</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Content</span>
</span></code></pre></div></td></tr></table></div>
<div class="doc doc-contents ">
<p>Fetch raw content from the given source.</p>
<p>Implementations must retrieve the content referenced by <code>source</code>
and return it as raw bytes wrapped in a <code>Content</code> object.</p>
<p><span class="doc-section-title">Parameters:</span></p>
@@ -1043,14 +1036,14 @@ and return it as raw bytes wrapped in a <code>Content</code> object.</p>
<table>
<thead>
<tr>
<th>Type</th>
<th>Name</th> <th>Type</th>
<th>Description</th>
</tr>
</thead>
<tbody>
<tr class="doc-section-item">
<td>
<code><a class="autorefs autorefs-internal" title="omniread.core.content.Content" href="../content/#omniread.core.content.Content">Content</a></code>
<td><code>Content</code></td> <td>
<code><a class="autorefs autorefs-internal" title="omniread.core.content.Content" href="../../omniread/core/content/#omniread.core.content.Content">Content</a></code>
</td>
<td>
<div class="doc-md-description">
@@ -1058,42 +1051,6 @@ and return it as raw bytes wrapped in a <code>Content</code> object.</p>
</div>
</td>
</tr>
<tr class="doc-section-item">
<td>
<code><a class="autorefs autorefs-internal" title="omniread.core.content.Content" href="../content/#omniread.core.content.Content">Content</a></code>
</td>
<td>
<div class="doc-md-description">
<ul>
<li>Raw content bytes</li>
</ul>
</div>
</td>
</tr>
<tr class="doc-section-item">
<td>
<code><a class="autorefs autorefs-internal" title="omniread.core.content.Content" href="../content/#omniread.core.content.Content">Content</a></code>
</td>
<td>
<div class="doc-md-description">
<ul>
<li>Source identifier</li>
</ul>
</div>
</td>
</tr>
<tr class="doc-section-item">
<td>
<code><a class="autorefs autorefs-internal" title="omniread.core.content.Content" href="../content/#omniread.core.content.Content">Content</a></code>
</td>
<td>
<div class="doc-md-description">
<ul>
<li>Optional metadata</li>
</ul>
</div>
</td>
</tr>
</tbody>
</table>
@@ -1120,6 +1077,13 @@ and return it as raw bytes wrapped in a <code>Content</code> object.</p>
</tbody>
</table>
<details class="notes" open>
<summary>Notes</summary>
<p><strong>Responsibilities:</strong></p>
<div class="language-text highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">1</span></pre></div></td><td class="code"><div><pre><span></span><code>- Implementations must retrieve the content referenced by `source` and return it as raw bytes wrapped in a `Content` object
</code></pre></div></td></tr></table></div>
</details>
</div>
</div>
@@ -1157,6 +1121,8 @@ and return it as raw bytes wrapped in a <code>Content</code> object.</p>
</div>
<script>var tabs=__md_get("__tabs");if(Array.isArray(tabs))e:for(var set of document.querySelectorAll(".tabbed-set")){var labels=set.querySelector(".tabbed-labels");for(var tab of tabs)for(var label of labels.getElementsByTagName("label"))if(label.innerText.trim()===tab){var input=document.getElementById(label.htmlFor);input.checked=!0;continue e}}</script>
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
</div>
@@ -1194,7 +1160,7 @@ and return it as raw bytes wrapped in a <code>Content</code> object.</p>
<script id="__config" type="application/json">{"base": "../..", "features": ["navigation.tabs", "navigation.expand", "navigation.top", "navigation.instant", "content.code.copy", "content.code.annotate"], "search": "../../assets/javascripts/workers/search.973d3a69.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
<script id="__config" type="application/json">{"base": "../..", "features": ["navigation.sections", "navigation.expand", "navigation.top", "navigation.instant", "navigation.tracking", "navigation.indexes", "content.code.copy", "content.code.annotate", "content.tabs.link", "content.action.edit", "search.highlight", "search.share", "search.suggest"], "search": "../../assets/javascripts/workers/search.973d3a69.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
<script src="../../assets/javascripts/bundle.f55a23d4.min.js"></script>