Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| a91cb67487 | |||
| 650d3a19e8 | |||
| f92d40ed7f | |||
| 04d8e069d4 | |||
| c3f3c9e57b | |||
| de7d04eb1a | |||
| 0fbf0ca0f0 | |||
| 5842e6a227 | |||
| a188e78283 | |||
| 67a3074ab4 |
60
.drone.yml
60
.drone.yml
@@ -23,20 +23,50 @@ steps:
|
|||||||
commands:
|
commands:
|
||||||
- PACKAGE_NAME=$(grep -E '^name\s*=' pyproject.toml | head -1 | cut -d'"' -f2)
|
- PACKAGE_NAME=$(grep -E '^name\s*=' pyproject.toml | head -1 | cut -d'"' -f2)
|
||||||
- VERSION=$(grep -E '^version\s*=' pyproject.toml | head -1 | cut -d'"' -f2)
|
- VERSION=$(grep -E '^version\s*=' pyproject.toml | head -1 | cut -d'"' -f2)
|
||||||
- echo "🔍 Checking if $PACKAGE_NAME==$VERSION exists on $PIP_REPO_URL ..."
|
|
||||||
- |
|
- |
|
||||||
if curl -fsSL -u "$PIP_USERNAME:$PIP_PASSWORD" "$PIP_REPO_URL/simple/$PACKAGE_NAME/" | grep -q "$VERSION"; then
|
if [ "$DRONE_TAG" != "$VERSION" ]; then
|
||||||
echo "✅ $PACKAGE_NAME==$VERSION already exists — skipping build."
|
echo "Tag '$DRONE_TAG' does not match pyproject version '$VERSION'; refusing to publish."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
- echo "Checking if $PACKAGE_NAME==$VERSION exists on $PIP_REPO_URL ..."
|
||||||
|
- |
|
||||||
|
if curl -fsSL -u "$PIP_USERNAME:$PIP_PASSWORD" "$PIP_REPO_URL/simple/$PACKAGE_NAME/" | grep -qE "${PACKAGE_NAME}-${VERSION}(-|\.tar\.gz|\.whl)"; then
|
||||||
|
echo "$PACKAGE_NAME==$VERSION already exists; skipping build."
|
||||||
exit 78
|
exit 78
|
||||||
else
|
else
|
||||||
echo "🆕 New version detected: $PACKAGE_NAME==$VERSION"
|
echo "New version detected: $PACKAGE_NAME==$VERSION"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
- name: quality-gate
|
||||||
|
image: python:3.13-slim
|
||||||
|
environment:
|
||||||
|
PIP_REPO_URL:
|
||||||
|
from_secret: PIP_REPO_URL
|
||||||
|
PIP_USERNAME:
|
||||||
|
from_secret: PIP_USERNAME
|
||||||
|
PIP_PASSWORD:
|
||||||
|
from_secret: PIP_PASSWORD
|
||||||
|
commands:
|
||||||
|
- pip install --upgrade pip build
|
||||||
|
- |
|
||||||
|
if [ -z "$PIP_REPO_URL" ] || [ -z "$PIP_USERNAME" ] || [ -z "$PIP_PASSWORD" ]; then
|
||||||
|
echo "PIP_REPO_URL / PIP_USERNAME / PIP_PASSWORD are not set" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
- AUTH_URL="https://$PIP_USERNAME:$PIP_PASSWORD@$(echo "$PIP_REPO_URL" | sed -E 's|^[a-zA-Z]+://||; s|/*$||')/simple"
|
||||||
|
- pip install --index-url "$AUTH_URL" --extra-index-url https://pypi.org/simple/ -U ".[dev]" "pydoclint>=0.9.1"
|
||||||
|
- echo "Running quality gate..."
|
||||||
|
- python -m black --check .
|
||||||
|
- python -m ruff check .
|
||||||
|
- python -m mypy
|
||||||
|
- pydoclint .
|
||||||
|
- python -m pytest
|
||||||
|
|
||||||
- name: build-package
|
- name: build-package
|
||||||
image: python:3.13-slim
|
image: python:3.13-slim
|
||||||
commands:
|
commands:
|
||||||
- pip install --upgrade pip build
|
- pip install --upgrade pip build
|
||||||
- echo "📦 Building Python package..."
|
- echo "Building Python package..."
|
||||||
- python -m build
|
- python -m build
|
||||||
- ls -l dist
|
- ls -l dist
|
||||||
|
|
||||||
@@ -51,7 +81,7 @@ steps:
|
|||||||
from_secret: PIP_PASSWORD
|
from_secret: PIP_PASSWORD
|
||||||
commands:
|
commands:
|
||||||
- pip install --upgrade twine
|
- pip install --upgrade twine
|
||||||
- echo "🚀 Uploading to private PyPI at $PIP_REPO_URL ..."
|
- echo "Uploading to private PyPI at $PIP_REPO_URL ..."
|
||||||
- |
|
- |
|
||||||
twine upload \
|
twine upload \
|
||||||
--repository-url "$PIP_REPO_URL" \
|
--repository-url "$PIP_REPO_URL" \
|
||||||
@@ -98,25 +128,31 @@ steps:
|
|||||||
set -e
|
set -e
|
||||||
|
|
||||||
PACKAGE_NAME=$(grep -E '^name\s*=' pyproject.toml | cut -d'"' -f2)
|
PACKAGE_NAME=$(grep -E '^name\s*=' pyproject.toml | cut -d'"' -f2)
|
||||||
echo "📦 Package: $PACKAGE_NAME"
|
echo "Package: $PACKAGE_NAME"
|
||||||
|
|
||||||
for TAG in $(git tag --sort=version:refname); do
|
for TAG in $(git tag --sort=version:refname); do
|
||||||
VERSION="$TAG"
|
VERSION="$TAG"
|
||||||
echo "🔁 Version: $VERSION"
|
echo "Version: $VERSION"
|
||||||
|
|
||||||
if curl -fsSL -u "$PIP_USERNAME:$PIP_PASSWORD" \
|
if curl -fsSL -u "$PIP_USERNAME:$PIP_PASSWORD" \
|
||||||
"$PIP_REPO_URL/simple/$PACKAGE_NAME/" | grep -q "$VERSION"; then
|
"$PIP_REPO_URL/simple/$PACKAGE_NAME/" | grep -qE "${PACKAGE_NAME}-${VERSION}(-|\.tar\.gz|\.whl)"; then
|
||||||
echo "⏭️ Exists, skipping"
|
echo "Exists, skipping"
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
|
|
||||||
git checkout --force "$TAG"
|
git checkout --force "$TAG"
|
||||||
|
|
||||||
echo "🏗️ Building $VERSION"
|
TAG_VERSION=$(grep -E '^version\s*=' pyproject.toml | head -1 | cut -d'"' -f2)
|
||||||
|
if [ "$TAG" != "$TAG_VERSION" ]; then
|
||||||
|
echo "Tag $TAG != pyproject version $TAG_VERSION at that commit; skipping"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Building $VERSION"
|
||||||
rm -rf dist
|
rm -rf dist
|
||||||
python -m build
|
python -m build
|
||||||
|
|
||||||
echo "⬆️ Uploading $VERSION"
|
echo "Uploading $VERSION"
|
||||||
twine upload \
|
twine upload \
|
||||||
--repository-url "$PIP_REPO_URL" \
|
--repository-url "$PIP_REPO_URL" \
|
||||||
-u "$PIP_USERNAME" \
|
-u "$PIP_USERNAME" \
|
||||||
|
|||||||
7
.gitignore
vendored
7
.gitignore
vendored
@@ -25,10 +25,17 @@ env/
|
|||||||
htmlcov/
|
htmlcov/
|
||||||
coverage.xml
|
coverage.xml
|
||||||
|
|
||||||
|
# Lint / type checking
|
||||||
|
.mypy_cache/
|
||||||
|
.ruff_cache/
|
||||||
|
|
||||||
# Environment variables
|
# Environment variables
|
||||||
.env
|
.env
|
||||||
.env.local
|
.env.local
|
||||||
|
|
||||||
|
# Docs build artifacts
|
||||||
|
site
|
||||||
|
|
||||||
# OS
|
# OS
|
||||||
.DS_Store
|
.DS_Store
|
||||||
Thumbs.db
|
Thumbs.db
|
||||||
|
|||||||
32
CHANGELOG.md
Normal file
32
CHANGELOG.md
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
# Changelog
|
||||||
|
|
||||||
|
All notable changes to this project will be documented in this file.
|
||||||
|
|
||||||
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||||
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||||
|
|
||||||
|
## [Unreleased]
|
||||||
|
|
||||||
|
## [0.0.3] - 2026-10-17
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- `py.typed` marker for PEP 561 type information.
|
||||||
|
- Type stubs for the `csv` and `xlsx` subpackages.
|
||||||
|
- Hand-written wiki, plus doc-forge-generated API reference under `docs/lib`
|
||||||
|
and MCP bundle under `docs/mcp` (per-kind MkDocs configs).
|
||||||
|
- MIT `LICENSE`.
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
- Adopted the standard Type A PyPI CI pipeline: a `tag == version` guard and a
|
||||||
|
quality gate (black, ruff, mypy, pydoclint, pytest).
|
||||||
|
- Standardized `pyproject.toml` (canonical packaging, lint tool config, extras)
|
||||||
|
with a shared Google-style `[tool.pydoclint]` configuration.
|
||||||
|
- Docstring types modernised (`List` -> `list`, `Optional` -> `X | None`) to
|
||||||
|
match signatures.
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- `XLSX`/`CSV` members added to the `content` type surface.
|
||||||
|
- Stub fixes for typed API surfaces.
|
||||||
|
|
||||||
|
[Unreleased]: https://git.aetoskia.com/aetos/omniread/compare/0.0.3...HEAD
|
||||||
|
[0.0.3]: https://git.aetoskia.com/aetos/omniread/compare/0.0.2...0.0.3
|
||||||
21
LICENSE
Normal file
21
LICENSE
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2026 Aetoskia Platform
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
111
README.md
Normal file
111
README.md
Normal file
@@ -0,0 +1,111 @@
|
|||||||
|
# omniread
|
||||||
|
|
||||||
|
# Summary
|
||||||
|
|
||||||
|
`OmniRead` — format-agnostic content acquisition and parsing framework.
|
||||||
|
|
||||||
|
`OmniRead` provides a **cleanly layered architecture** for fetching, parsing,
|
||||||
|
and normalizing content from heterogeneous sources such as HTML documents
|
||||||
|
and PDF files.
|
||||||
|
|
||||||
|
The library is structured around three core concepts:
|
||||||
|
|
||||||
|
1. **`Content`**: A canonical, format-agnostic container representing raw content
|
||||||
|
bytes and minimal contextual metadata.
|
||||||
|
2. **`Scrapers`**: Components responsible for *acquiring* raw content from a
|
||||||
|
source (HTTP, filesystem, object storage, etc.). `Scrapers` never interpret
|
||||||
|
content.
|
||||||
|
3. **`Parsers`**: Components responsible for *interpreting* acquired content and
|
||||||
|
converting it into structured, typed representations.
|
||||||
|
|
||||||
|
`OmniRead` deliberately separates these responsibilities to ensure:
|
||||||
|
|
||||||
|
- Clear boundaries between IO and interpretation.
|
||||||
|
- Replaceable implementations per format.
|
||||||
|
- Predictable, testable behavior.
|
||||||
|
|
||||||
|
# Installation
|
||||||
|
|
||||||
|
Install `OmniRead` using pip:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pip install omniread
|
||||||
|
```
|
||||||
|
|
||||||
|
Install OmniRead using Poetry:
|
||||||
|
```bash
|
||||||
|
poetry add omniread
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Quick start
|
||||||
|
|
||||||
|
Example:
|
||||||
|
HTML example:
|
||||||
|
```python
|
||||||
|
from omniread import HTMLScraper, HTMLParser
|
||||||
|
|
||||||
|
scraper = HTMLScraper()
|
||||||
|
content = scraper.fetch("https://example.com")
|
||||||
|
|
||||||
|
class TitleParser(HTMLParser[str]):
|
||||||
|
def parse(self) -> str:
|
||||||
|
return self._soup.title.string
|
||||||
|
|
||||||
|
parser = TitleParser(content)
|
||||||
|
title = parser.parse()
|
||||||
|
```
|
||||||
|
|
||||||
|
PDF example:
|
||||||
|
```python
|
||||||
|
from omniread import FileSystemPDFClient, PDFScraper, PDFParser
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
client = FileSystemPDFClient()
|
||||||
|
scraper = PDFScraper(client=client)
|
||||||
|
content = scraper.fetch(Path("document.pdf"))
|
||||||
|
|
||||||
|
class TextPDFParser(PDFParser[str]):
|
||||||
|
def parse(self) -> str:
|
||||||
|
# implement PDF text extraction
|
||||||
|
...
|
||||||
|
|
||||||
|
parser = TextPDFParser(content)
|
||||||
|
result = parser.parse()
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Public API
|
||||||
|
|
||||||
|
This module re-exports the **recommended public entry points** of OmniRead.
|
||||||
|
Consumers are encouraged to import from this namespace rather than from
|
||||||
|
format-specific submodules directly, unless advanced customization is
|
||||||
|
required.
|
||||||
|
|
||||||
|
- `Content`: Canonical content model.
|
||||||
|
- `ContentType`: Supported media types.
|
||||||
|
- `HTMLScraper`: HTTP-based HTML acquisition.
|
||||||
|
- `HTMLParser`: Base parser for HTML DOM interpretation.
|
||||||
|
- `FileSystemPDFClient`: Local filesystem PDF access.
|
||||||
|
- `PDFScraper`: PDF-specific content acquisition.
|
||||||
|
- `PDFParser`: Base parser for PDF binary interpretation.
|
||||||
|
- `FileSystemXlsxClient`: Local filesystem spreadsheet access.
|
||||||
|
- `XlsxScraper`: XLSX-specific content acquisition.
|
||||||
|
- `XlsxParser`: Generic string-row parser for xlsx workbooks.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Core Philosophy
|
||||||
|
|
||||||
|
`OmniRead` is designed as a **decoupled content engine**:
|
||||||
|
|
||||||
|
1. **Separation of Concerns**: Scrapers *fetch*, Parsers *interpret*. Neither
|
||||||
|
knows about the other.
|
||||||
|
2. **Normalized Exchange**: All components communicate via the `Content` model,
|
||||||
|
ensuring a consistent contract.
|
||||||
|
3. **Format Agnosticism**: The core logic is independent of whether the input
|
||||||
|
is HTML, PDF, or JSON.
|
||||||
|
|
||||||
|
---
|
||||||
31
docforge.nav.yml
Normal file
31
docforge.nav.yml
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
home: lib/index.md
|
||||||
|
groups:
|
||||||
|
Core API:
|
||||||
|
- lib/core/index.md
|
||||||
|
- lib/core/content.md
|
||||||
|
- lib/core/parser.md
|
||||||
|
- lib/core/scraper.md
|
||||||
|
HTML Handling:
|
||||||
|
- lib/html/index.md
|
||||||
|
- lib/html/parser.md
|
||||||
|
- lib/html/scraper.md
|
||||||
|
PDF Handling:
|
||||||
|
- lib/pdf/index.md
|
||||||
|
- lib/pdf/client.md
|
||||||
|
- lib/pdf/parser.md
|
||||||
|
- lib/pdf/scraper.md
|
||||||
|
CSV Handling:
|
||||||
|
- lib/csv/index.md
|
||||||
|
- lib/csv/client.md
|
||||||
|
- lib/csv/parser_base.md
|
||||||
|
- lib/csv/parser.md
|
||||||
|
- lib/csv/scraper.md
|
||||||
|
XLSX Handling:
|
||||||
|
- lib/xlsx/index.md
|
||||||
|
- lib/xlsx/client.md
|
||||||
|
- lib/xlsx/parser_base.md
|
||||||
|
- lib/xlsx/parser.md
|
||||||
|
- lib/xlsx/scraper.md
|
||||||
|
icon:
|
||||||
|
logo: material/book-open-page-variant
|
||||||
|
repo: fontawesome/brands/github
|
||||||
@@ -1 +1,3 @@
|
|||||||
|
# Content
|
||||||
|
|
||||||
::: omniread.core.content
|
::: omniread.core.content
|
||||||
@@ -1 +1,3 @@
|
|||||||
|
# Core
|
||||||
|
|
||||||
::: omniread.core
|
::: omniread.core
|
||||||
@@ -1 +1,3 @@
|
|||||||
|
# Parser
|
||||||
|
|
||||||
::: omniread.core.parser
|
::: omniread.core.parser
|
||||||
@@ -1 +1,3 @@
|
|||||||
|
# Scraper
|
||||||
|
|
||||||
::: omniread.core.scraper
|
::: omniread.core.scraper
|
||||||
3
docs/lib/csv/client.md
Normal file
3
docs/lib/csv/client.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Client
|
||||||
|
|
||||||
|
::: omniread.csv.client
|
||||||
3
docs/lib/csv/index.md
Normal file
3
docs/lib/csv/index.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Csv
|
||||||
|
|
||||||
|
::: omniread.csv
|
||||||
3
docs/lib/csv/parser.md
Normal file
3
docs/lib/csv/parser.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Parser
|
||||||
|
|
||||||
|
::: omniread.csv.parser
|
||||||
3
docs/lib/csv/parser_base.md
Normal file
3
docs/lib/csv/parser_base.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Parser Base
|
||||||
|
|
||||||
|
::: omniread.csv.parser_base
|
||||||
3
docs/lib/csv/scraper.md
Normal file
3
docs/lib/csv/scraper.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Scraper
|
||||||
|
|
||||||
|
::: omniread.csv.scraper
|
||||||
@@ -1 +1,3 @@
|
|||||||
|
# Html
|
||||||
|
|
||||||
::: omniread.html
|
::: omniread.html
|
||||||
@@ -1 +1,3 @@
|
|||||||
|
# Parser
|
||||||
|
|
||||||
::: omniread.html.parser
|
::: omniread.html.parser
|
||||||
@@ -1 +1,3 @@
|
|||||||
|
# Scraper
|
||||||
|
|
||||||
::: omniread.html.scraper
|
::: omniread.html.scraper
|
||||||
@@ -1 +1,3 @@
|
|||||||
|
# omniread
|
||||||
|
|
||||||
::: omniread
|
::: omniread
|
||||||
@@ -1 +1,3 @@
|
|||||||
|
# Client
|
||||||
|
|
||||||
::: omniread.pdf.client
|
::: omniread.pdf.client
|
||||||
@@ -1 +1,3 @@
|
|||||||
|
# Pdf
|
||||||
|
|
||||||
::: omniread.pdf
|
::: omniread.pdf
|
||||||
@@ -1 +1,3 @@
|
|||||||
|
# Parser
|
||||||
|
|
||||||
::: omniread.pdf.parser
|
::: omniread.pdf.parser
|
||||||
@@ -1 +1,3 @@
|
|||||||
|
# Scraper
|
||||||
|
|
||||||
::: omniread.pdf.scraper
|
::: omniread.pdf.scraper
|
||||||
3
docs/lib/xlsx/client.md
Normal file
3
docs/lib/xlsx/client.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Client
|
||||||
|
|
||||||
|
::: omniread.xlsx.client
|
||||||
3
docs/lib/xlsx/index.md
Normal file
3
docs/lib/xlsx/index.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Xlsx
|
||||||
|
|
||||||
|
::: omniread.xlsx
|
||||||
3
docs/lib/xlsx/parser.md
Normal file
3
docs/lib/xlsx/parser.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Parser
|
||||||
|
|
||||||
|
::: omniread.xlsx.parser
|
||||||
3
docs/lib/xlsx/parser_base.md
Normal file
3
docs/lib/xlsx/parser_base.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Parser Base
|
||||||
|
|
||||||
|
::: omniread.xlsx.parser_base
|
||||||
3
docs/lib/xlsx/scraper.md
Normal file
3
docs/lib/xlsx/scraper.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Scraper
|
||||||
|
|
||||||
|
::: omniread.xlsx.scraper
|
||||||
6
docs/mcp/index.json
Normal file
6
docs/mcp/index.json
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
{
|
||||||
|
"project": "OmniRead",
|
||||||
|
"type": "docforge-model",
|
||||||
|
"modules_count": 22,
|
||||||
|
"source": "docforge"
|
||||||
|
}
|
||||||
97
docs/mcp/modules/omniread.core.content.json
Normal file
97
docs/mcp/modules/omniread.core.content.json
Normal file
@@ -0,0 +1,97 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.core.content",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.core.content",
|
||||||
|
"docstring": "# Summary\n\nCanonical content models for OmniRead.\n\nThis module defines the **format-agnostic content representation** used across\nall parsers and scrapers in OmniRead.\n\nThe models defined here represent *what* was extracted, not *how* it was\nretrieved or parsed. Format-specific behavior and metadata must not alter\nthe semantic meaning of these models.",
|
||||||
|
"objects": {
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.core.content.ContentType",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.content.ContentType.HTML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.content.ContentType.PDF",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.content.ContentType.XLSX",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.content.ContentType.CSV",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.content.ContentType.JSON",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.content.ContentType.XML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.core.content.Content",
|
||||||
|
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.content.Content.raw",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.content.Content.source",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.content.Content.content_type",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.content.Content.metadata",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
450
docs/mcp/modules/omniread.core.json
Normal file
450
docs/mcp/modules/omniread.core.json
Normal file
@@ -0,0 +1,450 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.core",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.core",
|
||||||
|
"docstring": "# Summary\n\nCore domain contracts for OmniRead.\n\nThis package defines the **format-agnostic domain layer** of OmniRead.\nIt exposes canonical content models and abstract interfaces that are\nimplemented by format-specific modules (HTML, PDF, etc.).\n\nPublic exports from this package are considered **stable contracts** and\nare safe for downstream consumers to depend on.\n\nSubmodules:\n\n- `content`: Canonical content models and enums.\n- `parser`: Abstract parsing contracts.\n- `scraper`: Abstract scraping contracts.\n\nFormat-specific behavior must not be introduced at this layer.\n\n---\n\n# Public API\n\n- `Content`\n- `ContentType`\n\n---",
|
||||||
|
"objects": {
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.core.Content",
|
||||||
|
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.Content.raw",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.Content.source",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.Content.content_type",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.Content.metadata",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.core.ContentType",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.ContentType.HTML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.ContentType.PDF",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.ContentType.XLSX",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.ContentType.CSV",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.ContentType.JSON",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.ContentType.XML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseParser": {
|
||||||
|
"name": "BaseParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.core.BaseParser",
|
||||||
|
"signature": "BaseParser(content: Content)",
|
||||||
|
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.BaseParser.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic."
|
||||||
|
},
|
||||||
|
"content": {
|
||||||
|
"name": "content",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.BaseParser.content",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.core.BaseParser.parse",
|
||||||
|
"signature": "parse()",
|
||||||
|
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
|
||||||
|
},
|
||||||
|
"supports": {
|
||||||
|
"name": "supports",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.core.BaseParser.supports",
|
||||||
|
"signature": "supports()",
|
||||||
|
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseScraper": {
|
||||||
|
"name": "BaseScraper",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.core.BaseScraper",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.core.BaseScraper.fetch",
|
||||||
|
"signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None)",
|
||||||
|
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"content": {
|
||||||
|
"name": "content",
|
||||||
|
"kind": "module",
|
||||||
|
"path": "omniread.core.content",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "# Summary\n\nCanonical content models for OmniRead.\n\nThis module defines the **format-agnostic content representation** used across\nall parsers and scrapers in OmniRead.\n\nThe models defined here represent *what* was extracted, not *how* it was\nretrieved or parsed. Format-specific behavior and metadata must not alter\nthe semantic meaning of these models.",
|
||||||
|
"members": {
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.core.content.ContentType",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.content.ContentType.HTML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.content.ContentType.PDF",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.content.ContentType.XLSX",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.content.ContentType.CSV",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.content.ContentType.JSON",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.content.ContentType.XML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.core.content.Content",
|
||||||
|
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.content.Content.raw",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.content.Content.source",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.content.Content.content_type",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.content.Content.metadata",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"parser": {
|
||||||
|
"name": "parser",
|
||||||
|
"kind": "module",
|
||||||
|
"path": "omniread.core.parser",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "# Summary\n\nAbstract parsing contracts for OmniRead.\n\nThis module defines the **format-agnostic parser interface** used to transform\nraw content into structured, typed representations.\n\nParsers are responsible for:\n\n- Interpreting a single `Content` instance\n- Validating compatibility with the content type\n- Producing a structured output suitable for downstream consumers\n\nParsers are not responsible for:\n\n- Fetching or acquiring content\n- Performing retries or error recovery\n- Managing multiple content sources",
|
||||||
|
"members": {
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.core.parser.Content",
|
||||||
|
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.Content.raw",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.Content.source",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.Content.content_type",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.Content.metadata",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.core.parser.ContentType",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.ContentType.HTML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.ContentType.PDF",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.ContentType.XLSX",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.ContentType.CSV",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.ContentType.JSON",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.ContentType.XML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"T": {
|
||||||
|
"name": "T",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.T",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"BaseParser": {
|
||||||
|
"name": "BaseParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.core.parser.BaseParser",
|
||||||
|
"signature": "BaseParser(content: Content)",
|
||||||
|
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.BaseParser.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic."
|
||||||
|
},
|
||||||
|
"content": {
|
||||||
|
"name": "content",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.BaseParser.content",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.core.parser.BaseParser.parse",
|
||||||
|
"signature": "parse() -> T",
|
||||||
|
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
|
||||||
|
},
|
||||||
|
"supports": {
|
||||||
|
"name": "supports",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.core.parser.BaseParser.supports",
|
||||||
|
"signature": "supports() -> bool",
|
||||||
|
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"scraper": {
|
||||||
|
"name": "scraper",
|
||||||
|
"kind": "module",
|
||||||
|
"path": "omniread.core.scraper",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "# Summary\n\nAbstract scraping contracts for OmniRead.\n\nThis module defines the **format-agnostic scraper interface** responsible for\nacquiring raw content from external sources.\n\nScrapers are responsible for:\n\n- Locating and retrieving raw content bytes\n- Attaching minimal contextual metadata\n- Returning normalized `Content` objects\n\nScrapers are explicitly NOT responsible for:\n\n- Parsing or interpreting content\n- Inferring structure or semantics\n- Performing content-type specific processing\n\nAll interpretation must be delegated to parsers.",
|
||||||
|
"members": {
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.core.scraper.Content",
|
||||||
|
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.scraper.Content.raw",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.scraper.Content.source",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.scraper.Content.content_type",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.scraper.Content.metadata",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseScraper": {
|
||||||
|
"name": "BaseScraper",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.core.scraper.BaseScraper",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.core.scraper.BaseScraper.fetch",
|
||||||
|
"signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None) -> Content",
|
||||||
|
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
141
docs/mcp/modules/omniread.core.parser.json
Normal file
141
docs/mcp/modules/omniread.core.parser.json
Normal file
@@ -0,0 +1,141 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.core.parser",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.core.parser",
|
||||||
|
"docstring": "# Summary\n\nAbstract parsing contracts for OmniRead.\n\nThis module defines the **format-agnostic parser interface** used to transform\nraw content into structured, typed representations.\n\nParsers are responsible for:\n\n- Interpreting a single `Content` instance\n- Validating compatibility with the content type\n- Producing a structured output suitable for downstream consumers\n\nParsers are not responsible for:\n\n- Fetching or acquiring content\n- Performing retries or error recovery\n- Managing multiple content sources",
|
||||||
|
"objects": {
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.core.parser.Content",
|
||||||
|
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.Content.raw",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.Content.source",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.Content.content_type",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.Content.metadata",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.core.parser.ContentType",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.ContentType.HTML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.ContentType.PDF",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.ContentType.XLSX",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.ContentType.CSV",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.ContentType.JSON",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.ContentType.XML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"T": {
|
||||||
|
"name": "T",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.T",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"BaseParser": {
|
||||||
|
"name": "BaseParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.core.parser.BaseParser",
|
||||||
|
"signature": "BaseParser(content: Content)",
|
||||||
|
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.BaseParser.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic."
|
||||||
|
},
|
||||||
|
"content": {
|
||||||
|
"name": "content",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.BaseParser.content",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.core.parser.BaseParser.parse",
|
||||||
|
"signature": "parse() -> T",
|
||||||
|
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
|
||||||
|
},
|
||||||
|
"supports": {
|
||||||
|
"name": "supports",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.core.parser.BaseParser.supports",
|
||||||
|
"signature": "supports() -> bool",
|
||||||
|
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
62
docs/mcp/modules/omniread.core.scraper.json
Normal file
62
docs/mcp/modules/omniread.core.scraper.json
Normal file
@@ -0,0 +1,62 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.core.scraper",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.core.scraper",
|
||||||
|
"docstring": "# Summary\n\nAbstract scraping contracts for OmniRead.\n\nThis module defines the **format-agnostic scraper interface** responsible for\nacquiring raw content from external sources.\n\nScrapers are responsible for:\n\n- Locating and retrieving raw content bytes\n- Attaching minimal contextual metadata\n- Returning normalized `Content` objects\n\nScrapers are explicitly NOT responsible for:\n\n- Parsing or interpreting content\n- Inferring structure or semantics\n- Performing content-type specific processing\n\nAll interpretation must be delegated to parsers.",
|
||||||
|
"objects": {
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.core.scraper.Content",
|
||||||
|
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.scraper.Content.raw",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.scraper.Content.source",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.scraper.Content.content_type",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.scraper.Content.metadata",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseScraper": {
|
||||||
|
"name": "BaseScraper",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.core.scraper.BaseScraper",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.core.scraper.BaseScraper.fetch",
|
||||||
|
"signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None) -> Content",
|
||||||
|
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
41
docs/mcp/modules/omniread.csv.client.json
Normal file
41
docs/mcp/modules/omniread.csv.client.json
Normal file
@@ -0,0 +1,41 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.csv.client",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.csv.client",
|
||||||
|
"docstring": "# Summary\n\nCSV client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw\ncomma-separated-value document bytes from a concrete backing store.\n\nClients provide low-level access to csv binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems",
|
||||||
|
"objects": {
|
||||||
|
"BaseCsvClient": {
|
||||||
|
"name": "BaseCsvClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.client.BaseCsvClient",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Abstract client responsible for retrieving csv bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full csv binary payload.\n - Raise retrieval-specific errors on failure.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.client.BaseCsvClient.fetch",
|
||||||
|
"signature": "fetch(source: Any) -> bytes",
|
||||||
|
"docstring": "Fetch raw csv bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"FileSystemCsvClient": {
|
||||||
|
"name": "FileSystemCsvClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.client.FileSystemCsvClient",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "CSV client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads csv files directly from the disk and\n returns their raw binary contents.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.client.FileSystemCsvClient.fetch",
|
||||||
|
"signature": "fetch(path: Path) -> bytes",
|
||||||
|
"docstring": "Read a csv file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the csv file.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
492
docs/mcp/modules/omniread.csv.json
Normal file
492
docs/mcp/modules/omniread.csv.json
Normal file
@@ -0,0 +1,492 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.csv",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.csv",
|
||||||
|
"docstring": "# Summary\n\nCSV subpackage for OmniRead.\n\nProvides acquisition and parsing of comma-separated-value content:\n\n- `BaseCsvClient`: abstract backing-store client for csv bytes.\n- `FileSystemCsvClient`: local filesystem implementation.\n- `CsvScraper`: wraps fetched bytes into canonical `Content`.\n- `CsvParserBase`: content-type-enforcing parser contract.\n- `CsvParser`: generic string-row parser built on the standard csv module.",
|
||||||
|
"objects": {
|
||||||
|
"BaseCsvClient": {
|
||||||
|
"name": "BaseCsvClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.BaseCsvClient",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Abstract client responsible for retrieving csv bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full csv binary payload.\n - Raise retrieval-specific errors on failure.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.BaseCsvClient.fetch",
|
||||||
|
"signature": "fetch(source: Any)",
|
||||||
|
"docstring": "Fetch raw csv bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"FileSystemCsvClient": {
|
||||||
|
"name": "FileSystemCsvClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.FileSystemCsvClient",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "CSV client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads csv files directly from the disk and\n returns their raw binary contents.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.FileSystemCsvClient.fetch",
|
||||||
|
"signature": "fetch(path: Path)",
|
||||||
|
"docstring": "Read a csv file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the csv file.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"CsvParser": {
|
||||||
|
"name": "CsvParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.CsvParser",
|
||||||
|
"signature": "CsvParser(content: Content)",
|
||||||
|
"docstring": "Generic csv parser producing string rows from the document.\n\nNotes:\n **Responsibilities:**\n\n - Decode the payload (UTF-8 with BOM support, Latin-1 fallback).\n - Detect the delimiter from a leading sample (`,` `;` tab `|`),\n defaulting to `,`.\n - Normalize cells into deterministic stripped string values.\n - Expose row extraction helpers mirroring `XlsxParser.rows`.\n\n **Constraints:**\n\n - All values are strings; consumers requiring typed values must\n convert on their side.\n - Quoted fields containing delimiters/newlines are handled by\n the standard ``csv`` module.",
|
||||||
|
"members": {
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.CsvParser.parse",
|
||||||
|
"signature": "parse()",
|
||||||
|
"docstring": "Parse the document into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the document."
|
||||||
|
},
|
||||||
|
"rows": {
|
||||||
|
"name": "rows",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.CsvParser.rows",
|
||||||
|
"signature": "rows(*, skip_empty: bool = True)",
|
||||||
|
"docstring": "Extract normalized string rows from the document.\n\nArgs:\n skip_empty (bool):\n When True (default), rows whose cells are all blank are\n omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"CsvParserBase": {
|
||||||
|
"name": "CsvParserBase",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.CsvParserBase",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Base csv parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces csv content-type compatibility and provides\n the extension point for implementing concrete csv parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.CsvParserBase.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser (CSV only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.CsvParserBase.parse",
|
||||||
|
"signature": "parse()",
|
||||||
|
"docstring": "Parse csv content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"CsvScraper": {
|
||||||
|
"name": "CsvScraper",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.CsvScraper",
|
||||||
|
"signature": "CsvScraper(*, client: BaseCsvClient)",
|
||||||
|
"docstring": "Scraper for csv documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw csv bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n CSV content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.CsvScraper.fetch",
|
||||||
|
"signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None)",
|
||||||
|
"docstring": "Fetch a csv document from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw csv bytes, source\n identifier, CSV content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"client": {
|
||||||
|
"name": "client",
|
||||||
|
"kind": "module",
|
||||||
|
"path": "omniread.csv.client",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "# Summary\n\nCSV client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw\ncomma-separated-value document bytes from a concrete backing store.\n\nClients provide low-level access to csv binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems",
|
||||||
|
"members": {
|
||||||
|
"BaseCsvClient": {
|
||||||
|
"name": "BaseCsvClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.client.BaseCsvClient",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Abstract client responsible for retrieving csv bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full csv binary payload.\n - Raise retrieval-specific errors on failure.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.client.BaseCsvClient.fetch",
|
||||||
|
"signature": "fetch(source: Any) -> bytes",
|
||||||
|
"docstring": "Fetch raw csv bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"FileSystemCsvClient": {
|
||||||
|
"name": "FileSystemCsvClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.client.FileSystemCsvClient",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "CSV client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads csv files directly from the disk and\n returns their raw binary contents.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.client.FileSystemCsvClient.fetch",
|
||||||
|
"signature": "fetch(path: Path) -> bytes",
|
||||||
|
"docstring": "Read a csv file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the csv file.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"parser": {
|
||||||
|
"name": "parser",
|
||||||
|
"kind": "module",
|
||||||
|
"path": "omniread.csv.parser",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "# Summary\n\nCSV parser implementations for OmniRead.\n\nThis module provides a concrete, generic parser for comma-separated-value\ndocuments. It exposes records as lists of string cells so downstream\nconsumers can interpret tabular content without depending on the ``csv``\nmodule directly.\n\nThe parser is intentionally statement-agnostic: it performs no header\ndetection or column interpretation beyond basic cell normalization and\ndelimiter detection.",
|
||||||
|
"members": {
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.parser.Content",
|
||||||
|
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser.Content.raw",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser.Content.source",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser.Content.content_type",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser.Content.metadata",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"CsvParserBase": {
|
||||||
|
"name": "CsvParserBase",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.parser.CsvParserBase",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Base csv parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces csv content-type compatibility and provides\n the extension point for implementing concrete csv parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser.CsvParserBase.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser (CSV only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.parser.CsvParserBase.parse",
|
||||||
|
"signature": "parse()",
|
||||||
|
"docstring": "Parse csv content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"CsvParser": {
|
||||||
|
"name": "CsvParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.parser.CsvParser",
|
||||||
|
"signature": "CsvParser(content: Content)",
|
||||||
|
"docstring": "Generic csv parser producing string rows from the document.\n\nNotes:\n **Responsibilities:**\n\n - Decode the payload (UTF-8 with BOM support, Latin-1 fallback).\n - Detect the delimiter from a leading sample (`,` `;` tab `|`),\n defaulting to `,`.\n - Normalize cells into deterministic stripped string values.\n - Expose row extraction helpers mirroring `XlsxParser.rows`.\n\n **Constraints:**\n\n - All values are strings; consumers requiring typed values must\n convert on their side.\n - Quoted fields containing delimiters/newlines are handled by\n the standard ``csv`` module.",
|
||||||
|
"members": {
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.parser.CsvParser.parse",
|
||||||
|
"signature": "parse() -> list[list[str]]",
|
||||||
|
"docstring": "Parse the document into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the document."
|
||||||
|
},
|
||||||
|
"rows": {
|
||||||
|
"name": "rows",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.parser.CsvParser.rows",
|
||||||
|
"signature": "rows(*, skip_empty: bool = True) -> list[list[str]]",
|
||||||
|
"docstring": "Extract normalized string rows from the document.\n\nArgs:\n skip_empty (bool):\n When True (default), rows whose cells are all blank are\n omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"parser_base": {
|
||||||
|
"name": "parser_base",
|
||||||
|
"kind": "module",
|
||||||
|
"path": "omniread.csv.parser_base",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "# Summary\n\nCSV parser base implementation for OmniRead.\n\nThis module defines the **CSV-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for\ncomma-separated-value documents.",
|
||||||
|
"members": {
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType.HTML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType.PDF",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType.XLSX",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType.CSV",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType.JSON",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType.XML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseParser": {
|
||||||
|
"name": "BaseParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.parser_base.BaseParser",
|
||||||
|
"signature": "BaseParser(content: Content)",
|
||||||
|
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.BaseParser.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic."
|
||||||
|
},
|
||||||
|
"content": {
|
||||||
|
"name": "content",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.BaseParser.content",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.parser_base.BaseParser.parse",
|
||||||
|
"signature": "parse()",
|
||||||
|
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
|
||||||
|
},
|
||||||
|
"supports": {
|
||||||
|
"name": "supports",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.parser_base.BaseParser.supports",
|
||||||
|
"signature": "supports()",
|
||||||
|
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"T": {
|
||||||
|
"name": "T",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.T",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"CsvParserBase": {
|
||||||
|
"name": "CsvParserBase",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.parser_base.CsvParserBase",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Base csv parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces csv content-type compatibility and provides\n the extension point for implementing concrete csv parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.CsvParserBase.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser (CSV only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.parser_base.CsvParserBase.parse",
|
||||||
|
"signature": "parse() -> T",
|
||||||
|
"docstring": "Parse csv content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"scraper": {
|
||||||
|
"name": "scraper",
|
||||||
|
"kind": "module",
|
||||||
|
"path": "omniread.csv.scraper",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "# Summary\n\nCSV scraper for OmniRead.\n\nThis module defines the scraper responsible for acquiring raw\ncomma-separated-value document content from a backing store via a\nconfigured client.\n\nThe scraper does not interpret or parse the acquired bytes; it wraps them in\nthe canonical `Content` model.",
|
||||||
|
"members": {
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.scraper.Content",
|
||||||
|
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.Content.raw",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.Content.source",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.Content.content_type",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.Content.metadata",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.scraper.ContentType",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.ContentType.HTML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.ContentType.PDF",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.ContentType.XLSX",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.ContentType.CSV",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.ContentType.JSON",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.ContentType.XML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseCsvClient": {
|
||||||
|
"name": "BaseCsvClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.scraper.BaseCsvClient",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Abstract client responsible for retrieving csv bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full csv binary payload.\n - Raise retrieval-specific errors on failure.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.scraper.BaseCsvClient.fetch",
|
||||||
|
"signature": "fetch(source: Any)",
|
||||||
|
"docstring": "Fetch raw csv bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"CsvScraper": {
|
||||||
|
"name": "CsvScraper",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.scraper.CsvScraper",
|
||||||
|
"signature": "CsvScraper(*, client: BaseCsvClient)",
|
||||||
|
"docstring": "Scraper for csv documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw csv bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n CSV content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.scraper.CsvScraper.fetch",
|
||||||
|
"signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None) -> Content",
|
||||||
|
"docstring": "Fetch a csv document from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw csv bytes, source\n identifier, CSV content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
92
docs/mcp/modules/omniread.csv.parser.json
Normal file
92
docs/mcp/modules/omniread.csv.parser.json
Normal file
@@ -0,0 +1,92 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.csv.parser",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.csv.parser",
|
||||||
|
"docstring": "# Summary\n\nCSV parser implementations for OmniRead.\n\nThis module provides a concrete, generic parser for comma-separated-value\ndocuments. It exposes records as lists of string cells so downstream\nconsumers can interpret tabular content without depending on the ``csv``\nmodule directly.\n\nThe parser is intentionally statement-agnostic: it performs no header\ndetection or column interpretation beyond basic cell normalization and\ndelimiter detection.",
|
||||||
|
"objects": {
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.parser.Content",
|
||||||
|
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser.Content.raw",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser.Content.source",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser.Content.content_type",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser.Content.metadata",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"CsvParserBase": {
|
||||||
|
"name": "CsvParserBase",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.parser.CsvParserBase",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Base csv parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces csv content-type compatibility and provides\n the extension point for implementing concrete csv parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser.CsvParserBase.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser (CSV only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.parser.CsvParserBase.parse",
|
||||||
|
"signature": "parse()",
|
||||||
|
"docstring": "Parse csv content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"CsvParser": {
|
||||||
|
"name": "CsvParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.parser.CsvParser",
|
||||||
|
"signature": "CsvParser(content: Content)",
|
||||||
|
"docstring": "Generic csv parser producing string rows from the document.\n\nNotes:\n **Responsibilities:**\n\n - Decode the payload (UTF-8 with BOM support, Latin-1 fallback).\n - Detect the delimiter from a leading sample (`,` `;` tab `|`),\n defaulting to `,`.\n - Normalize cells into deterministic stripped string values.\n - Expose row extraction helpers mirroring `XlsxParser.rows`.\n\n **Constraints:**\n\n - All values are strings; consumers requiring typed values must\n convert on their side.\n - Quoted fields containing delimiters/newlines are handled by\n the standard ``csv`` module.",
|
||||||
|
"members": {
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.parser.CsvParser.parse",
|
||||||
|
"signature": "parse() -> list[list[str]]",
|
||||||
|
"docstring": "Parse the document into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the document."
|
||||||
|
},
|
||||||
|
"rows": {
|
||||||
|
"name": "rows",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.parser.CsvParser.rows",
|
||||||
|
"signature": "rows(*, skip_empty: bool = True) -> list[list[str]]",
|
||||||
|
"docstring": "Extract normalized string rows from the document.\n\nArgs:\n skip_empty (bool):\n When True (default), rows whose cells are all blank are\n omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
127
docs/mcp/modules/omniread.csv.parser_base.json
Normal file
127
docs/mcp/modules/omniread.csv.parser_base.json
Normal file
@@ -0,0 +1,127 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.csv.parser_base",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.csv.parser_base",
|
||||||
|
"docstring": "# Summary\n\nCSV parser base implementation for OmniRead.\n\nThis module defines the **CSV-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for\ncomma-separated-value documents.",
|
||||||
|
"objects": {
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType.HTML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType.PDF",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType.XLSX",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType.CSV",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType.JSON",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType.XML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseParser": {
|
||||||
|
"name": "BaseParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.parser_base.BaseParser",
|
||||||
|
"signature": "BaseParser(content: Content)",
|
||||||
|
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.BaseParser.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic."
|
||||||
|
},
|
||||||
|
"content": {
|
||||||
|
"name": "content",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.BaseParser.content",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.parser_base.BaseParser.parse",
|
||||||
|
"signature": "parse()",
|
||||||
|
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
|
||||||
|
},
|
||||||
|
"supports": {
|
||||||
|
"name": "supports",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.parser_base.BaseParser.supports",
|
||||||
|
"signature": "supports()",
|
||||||
|
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"T": {
|
||||||
|
"name": "T",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.T",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"CsvParserBase": {
|
||||||
|
"name": "CsvParserBase",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.parser_base.CsvParserBase",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Base csv parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces csv content-type compatibility and provides\n the extension point for implementing concrete csv parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.CsvParserBase.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser (CSV only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.parser_base.CsvParserBase.parse",
|
||||||
|
"signature": "parse() -> T",
|
||||||
|
"docstring": "Parse csv content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
129
docs/mcp/modules/omniread.csv.scraper.json
Normal file
129
docs/mcp/modules/omniread.csv.scraper.json
Normal file
@@ -0,0 +1,129 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.csv.scraper",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.csv.scraper",
|
||||||
|
"docstring": "# Summary\n\nCSV scraper for OmniRead.\n\nThis module defines the scraper responsible for acquiring raw\ncomma-separated-value document content from a backing store via a\nconfigured client.\n\nThe scraper does not interpret or parse the acquired bytes; it wraps them in\nthe canonical `Content` model.",
|
||||||
|
"objects": {
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.scraper.Content",
|
||||||
|
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.Content.raw",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.Content.source",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.Content.content_type",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.Content.metadata",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.scraper.ContentType",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.ContentType.HTML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.ContentType.PDF",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.ContentType.XLSX",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.ContentType.CSV",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.ContentType.JSON",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.ContentType.XML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseCsvClient": {
|
||||||
|
"name": "BaseCsvClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.scraper.BaseCsvClient",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Abstract client responsible for retrieving csv bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full csv binary payload.\n - Raise retrieval-specific errors on failure.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.scraper.BaseCsvClient.fetch",
|
||||||
|
"signature": "fetch(source: Any)",
|
||||||
|
"docstring": "Fetch raw csv bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"CsvScraper": {
|
||||||
|
"name": "CsvScraper",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.scraper.CsvScraper",
|
||||||
|
"signature": "CsvScraper(*, client: BaseCsvClient)",
|
||||||
|
"docstring": "Scraper for csv documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw csv bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n CSV content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.scraper.CsvScraper.fetch",
|
||||||
|
"signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None) -> Content",
|
||||||
|
"docstring": "Fetch a csv document from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw csv bytes, source\n identifier, CSV content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
425
docs/mcp/modules/omniread.html.json
Normal file
425
docs/mcp/modules/omniread.html.json
Normal file
@@ -0,0 +1,425 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.html",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.html",
|
||||||
|
"docstring": "# Summary\n\nHTML format implementation for OmniRead.\n\nThis package provides **HTML-specific implementations** of the core OmniRead\ncontracts defined in `omniread.core`.\n\nIt includes:\n\n- HTML parsers that interpret HTML content.\n- HTML scrapers that retrieve HTML documents.\n\nKey characteristics:\n\n- Implements, but does not redefine, core contracts.\n- May contain HTML-specific behavior and edge-case handling.\n- Produces canonical content models defined in `omniread.core.content`.\n\nConsumers should depend on `omniread.core` interfaces wherever possible and\nuse this package only when HTML-specific behavior is required.\n\n---\n\n# Public API\n\n- `HTMLScraper`\n- `HTMLParser`\n\n---",
|
||||||
|
"objects": {
|
||||||
|
"HTMLScraper": {
|
||||||
|
"name": "HTMLScraper",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.html.HTMLScraper",
|
||||||
|
"signature": "HTMLScraper(*, client: httpx.Client | None = None, timeout: float = 15.0, headers: Mapping[str, str] | None = None, follow_redirects: bool = True)",
|
||||||
|
"docstring": "Base HTML scraper using `httpx`.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns\n them as raw content wrapped in a `Content` object.\n - Fetches raw bytes and metadata only.\n - The scraper uses `httpx.Client` for HTTP requests, enforces an\n HTML content type, and preserves HTTP response metadata.\n\n **Constraints:**\n\n - The scraper does not: Parse HTML, perform retries or backoff,\n handle non-HTML responses.",
|
||||||
|
"members": {
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.HTMLScraper.content_type",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"validate_content_type": {
|
||||||
|
"name": "validate_content_type",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.HTMLScraper.validate_content_type",
|
||||||
|
"signature": "validate_content_type(response: httpx.Response)",
|
||||||
|
"docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content."
|
||||||
|
},
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.HTMLScraper.fetch",
|
||||||
|
"signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None)",
|
||||||
|
"docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"HTMLParser": {
|
||||||
|
"name": "HTMLParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.html.HTMLParser",
|
||||||
|
"signature": "HTMLParser(content: Content, features: str = 'html.parser')",
|
||||||
|
"docstring": "Base HTML parser.\n\nNotes:\n **Responsibilities:**\n\n - This class extends the core `BaseParser` with HTML-specific behavior,\n including DOM parsing via BeautifulSoup and reusable extraction helpers.\n - Provides reusable helpers for HTML extraction. Concrete parsers must\n explicitly define the return type.\n\n **Guarantees:**\n\n - Accepts only HTML content.\n - Owns a parsed BeautifulSoup DOM tree.\n - Provides pure helper utilities for common HTML structures.\n\n **Constraints:**\n\n - Concrete subclasses must define the output type `T` and implement\n the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.HTMLParser.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser (HTML only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.HTMLParser.parse",
|
||||||
|
"signature": "parse()",
|
||||||
|
"docstring": "Fully parse the HTML content into structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the HTML DOM and return a\n deterministic, structured output."
|
||||||
|
},
|
||||||
|
"parse_div": {
|
||||||
|
"name": "parse_div",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.HTMLParser.parse_div",
|
||||||
|
"signature": "parse_div(div: Tag, *, separator: str = ' ')",
|
||||||
|
"docstring": "Extract normalized text from a `<div>` element.\n\nArgs:\n div (Tag):\n BeautifulSoup tag representing a `<div>`.\n separator (str, optional):\n String used to separate text nodes.\n\nReturns:\n str:\n Flattened, whitespace-normalized text content."
|
||||||
|
},
|
||||||
|
"parse_link": {
|
||||||
|
"name": "parse_link",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.HTMLParser.parse_link",
|
||||||
|
"signature": "parse_link(a: Tag)",
|
||||||
|
"docstring": "Extract the hyperlink reference from an `<a>` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n str | None:\n The value of the `href` attribute, or None if absent."
|
||||||
|
},
|
||||||
|
"parse_table": {
|
||||||
|
"name": "parse_table",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.HTMLParser.parse_table",
|
||||||
|
"signature": "parse_table(table: Tag)",
|
||||||
|
"docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table (Tag):\n BeautifulSoup tag representing a `<table>`.\n\nReturns:\n list[list[str]]:\n A list of rows, where each row is a list of cell text values."
|
||||||
|
},
|
||||||
|
"parse_meta": {
|
||||||
|
"name": "parse_meta",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.HTMLParser.parse_meta",
|
||||||
|
"signature": "parse_meta()",
|
||||||
|
"docstring": "Extract high-level metadata from the HTML document.\n\nReturns:\n dict[str, Any]:\n Dictionary containing extracted metadata.\n\nNotes:\n **Responsibilities:**\n\n - Extract high-level metadata from the HTML document.\n - This includes: Document title, `<meta>` tag name/property to\n content mappings."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"parser": {
|
||||||
|
"name": "parser",
|
||||||
|
"kind": "module",
|
||||||
|
"path": "omniread.html.parser",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "# Summary\n\nHTML parser base implementations for OmniRead.\n\nThis module provides reusable HTML parsing utilities built on top of\nthe abstract parser contracts defined in `omniread.core.parser`.\n\nIt supplies:\n\n- Content-type enforcement for HTML inputs\n- BeautifulSoup initialization and lifecycle management\n- Common helper methods for extracting structured data from HTML elements\n\nConcrete parsers must subclass `HTMLParser` and implement the `parse()` method\nto return a structured representation appropriate for their use case.",
|
||||||
|
"members": {
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.html.parser.Content",
|
||||||
|
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.Content.raw",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.Content.source",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.Content.content_type",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.Content.metadata",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.html.parser.ContentType",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.ContentType.HTML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.ContentType.PDF",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.ContentType.XLSX",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.ContentType.CSV",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.ContentType.JSON",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.ContentType.XML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseParser": {
|
||||||
|
"name": "BaseParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.html.parser.BaseParser",
|
||||||
|
"signature": "BaseParser(content: Content)",
|
||||||
|
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.BaseParser.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic."
|
||||||
|
},
|
||||||
|
"content": {
|
||||||
|
"name": "content",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.BaseParser.content",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.parser.BaseParser.parse",
|
||||||
|
"signature": "parse()",
|
||||||
|
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
|
||||||
|
},
|
||||||
|
"supports": {
|
||||||
|
"name": "supports",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.parser.BaseParser.supports",
|
||||||
|
"signature": "supports()",
|
||||||
|
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"T": {
|
||||||
|
"name": "T",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.T",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"HTMLParser": {
|
||||||
|
"name": "HTMLParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.html.parser.HTMLParser",
|
||||||
|
"signature": "HTMLParser(content: Content, features: str = 'html.parser')",
|
||||||
|
"docstring": "Base HTML parser.\n\nNotes:\n **Responsibilities:**\n\n - This class extends the core `BaseParser` with HTML-specific behavior,\n including DOM parsing via BeautifulSoup and reusable extraction helpers.\n - Provides reusable helpers for HTML extraction. Concrete parsers must\n explicitly define the return type.\n\n **Guarantees:**\n\n - Accepts only HTML content.\n - Owns a parsed BeautifulSoup DOM tree.\n - Provides pure helper utilities for common HTML structures.\n\n **Constraints:**\n\n - Concrete subclasses must define the output type `T` and implement\n the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.HTMLParser.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser (HTML only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.parser.HTMLParser.parse",
|
||||||
|
"signature": "parse() -> T",
|
||||||
|
"docstring": "Fully parse the HTML content into structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the HTML DOM and return a\n deterministic, structured output."
|
||||||
|
},
|
||||||
|
"parse_div": {
|
||||||
|
"name": "parse_div",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.parser.HTMLParser.parse_div",
|
||||||
|
"signature": "parse_div(div: Tag, *, separator: str = ' ') -> str",
|
||||||
|
"docstring": "Extract normalized text from a `<div>` element.\n\nArgs:\n div (Tag):\n BeautifulSoup tag representing a `<div>`.\n separator (str, optional):\n String used to separate text nodes.\n\nReturns:\n str:\n Flattened, whitespace-normalized text content."
|
||||||
|
},
|
||||||
|
"parse_link": {
|
||||||
|
"name": "parse_link",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.parser.HTMLParser.parse_link",
|
||||||
|
"signature": "parse_link(a: Tag) -> str | None",
|
||||||
|
"docstring": "Extract the hyperlink reference from an `<a>` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n str | None:\n The value of the `href` attribute, or None if absent."
|
||||||
|
},
|
||||||
|
"parse_table": {
|
||||||
|
"name": "parse_table",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.parser.HTMLParser.parse_table",
|
||||||
|
"signature": "parse_table(table: Tag) -> list[list[str]]",
|
||||||
|
"docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table (Tag):\n BeautifulSoup tag representing a `<table>`.\n\nReturns:\n list[list[str]]:\n A list of rows, where each row is a list of cell text values."
|
||||||
|
},
|
||||||
|
"parse_meta": {
|
||||||
|
"name": "parse_meta",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.parser.HTMLParser.parse_meta",
|
||||||
|
"signature": "parse_meta() -> dict[str, Any]",
|
||||||
|
"docstring": "Extract high-level metadata from the HTML document.\n\nReturns:\n dict[str, Any]:\n Dictionary containing extracted metadata.\n\nNotes:\n **Responsibilities:**\n\n - Extract high-level metadata from the HTML document.\n - This includes: Document title, `<meta>` tag name/property to\n content mappings."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"scraper": {
|
||||||
|
"name": "scraper",
|
||||||
|
"kind": "module",
|
||||||
|
"path": "omniread.html.scraper",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "# Summary\n\nHTML scraping implementation for OmniRead.\n\nThis module provides an HTTP-based scraper for retrieving HTML documents.\nIt implements the core `BaseScraper` contract using `httpx` as the transport\nlayer.\n\nThis scraper is responsible for:\n\n- Fetching raw HTML bytes over HTTP(S)\n- Validating response content type\n- Attaching HTTP metadata to the returned content\n\nThis scraper is not responsible for:\n\n- Parsing or interpreting HTML\n- Retrying failed requests\n- Managing crawl policies or rate limiting",
|
||||||
|
"members": {
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.html.scraper.Content",
|
||||||
|
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.Content.raw",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.Content.source",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.Content.content_type",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.Content.metadata",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.html.scraper.ContentType",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.ContentType.HTML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.ContentType.PDF",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.ContentType.XLSX",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.ContentType.CSV",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.ContentType.JSON",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.ContentType.XML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseScraper": {
|
||||||
|
"name": "BaseScraper",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.html.scraper.BaseScraper",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.scraper.BaseScraper.fetch",
|
||||||
|
"signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None)",
|
||||||
|
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"HTMLScraper": {
|
||||||
|
"name": "HTMLScraper",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.html.scraper.HTMLScraper",
|
||||||
|
"signature": "HTMLScraper(*, client: httpx.Client | None = None, timeout: float = 15.0, headers: Mapping[str, str] | None = None, follow_redirects: bool = True)",
|
||||||
|
"docstring": "Base HTML scraper using `httpx`.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns\n them as raw content wrapped in a `Content` object.\n - Fetches raw bytes and metadata only.\n - The scraper uses `httpx.Client` for HTTP requests, enforces an\n HTML content type, and preserves HTTP response metadata.\n\n **Constraints:**\n\n - The scraper does not: Parse HTML, perform retries or backoff,\n handle non-HTML responses.",
|
||||||
|
"members": {
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.HTMLScraper.content_type",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"validate_content_type": {
|
||||||
|
"name": "validate_content_type",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.scraper.HTMLScraper.validate_content_type",
|
||||||
|
"signature": "validate_content_type(response: httpx.Response) -> None",
|
||||||
|
"docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content."
|
||||||
|
},
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.scraper.HTMLScraper.fetch",
|
||||||
|
"signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None) -> Content",
|
||||||
|
"docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
192
docs/mcp/modules/omniread.html.parser.json
Normal file
192
docs/mcp/modules/omniread.html.parser.json
Normal file
@@ -0,0 +1,192 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.html.parser",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.html.parser",
|
||||||
|
"docstring": "# Summary\n\nHTML parser base implementations for OmniRead.\n\nThis module provides reusable HTML parsing utilities built on top of\nthe abstract parser contracts defined in `omniread.core.parser`.\n\nIt supplies:\n\n- Content-type enforcement for HTML inputs\n- BeautifulSoup initialization and lifecycle management\n- Common helper methods for extracting structured data from HTML elements\n\nConcrete parsers must subclass `HTMLParser` and implement the `parse()` method\nto return a structured representation appropriate for their use case.",
|
||||||
|
"objects": {
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.html.parser.Content",
|
||||||
|
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.Content.raw",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.Content.source",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.Content.content_type",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.Content.metadata",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.html.parser.ContentType",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.ContentType.HTML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.ContentType.PDF",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.ContentType.XLSX",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.ContentType.CSV",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.ContentType.JSON",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.ContentType.XML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseParser": {
|
||||||
|
"name": "BaseParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.html.parser.BaseParser",
|
||||||
|
"signature": "BaseParser(content: Content)",
|
||||||
|
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.BaseParser.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic."
|
||||||
|
},
|
||||||
|
"content": {
|
||||||
|
"name": "content",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.BaseParser.content",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.parser.BaseParser.parse",
|
||||||
|
"signature": "parse()",
|
||||||
|
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
|
||||||
|
},
|
||||||
|
"supports": {
|
||||||
|
"name": "supports",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.parser.BaseParser.supports",
|
||||||
|
"signature": "supports()",
|
||||||
|
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"T": {
|
||||||
|
"name": "T",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.T",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"HTMLParser": {
|
||||||
|
"name": "HTMLParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.html.parser.HTMLParser",
|
||||||
|
"signature": "HTMLParser(content: Content, features: str = 'html.parser')",
|
||||||
|
"docstring": "Base HTML parser.\n\nNotes:\n **Responsibilities:**\n\n - This class extends the core `BaseParser` with HTML-specific behavior,\n including DOM parsing via BeautifulSoup and reusable extraction helpers.\n - Provides reusable helpers for HTML extraction. Concrete parsers must\n explicitly define the return type.\n\n **Guarantees:**\n\n - Accepts only HTML content.\n - Owns a parsed BeautifulSoup DOM tree.\n - Provides pure helper utilities for common HTML structures.\n\n **Constraints:**\n\n - Concrete subclasses must define the output type `T` and implement\n the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.HTMLParser.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser (HTML only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.parser.HTMLParser.parse",
|
||||||
|
"signature": "parse() -> T",
|
||||||
|
"docstring": "Fully parse the HTML content into structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the HTML DOM and return a\n deterministic, structured output."
|
||||||
|
},
|
||||||
|
"parse_div": {
|
||||||
|
"name": "parse_div",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.parser.HTMLParser.parse_div",
|
||||||
|
"signature": "parse_div(div: Tag, *, separator: str = ' ') -> str",
|
||||||
|
"docstring": "Extract normalized text from a `<div>` element.\n\nArgs:\n div (Tag):\n BeautifulSoup tag representing a `<div>`.\n separator (str, optional):\n String used to separate text nodes.\n\nReturns:\n str:\n Flattened, whitespace-normalized text content."
|
||||||
|
},
|
||||||
|
"parse_link": {
|
||||||
|
"name": "parse_link",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.parser.HTMLParser.parse_link",
|
||||||
|
"signature": "parse_link(a: Tag) -> str | None",
|
||||||
|
"docstring": "Extract the hyperlink reference from an `<a>` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n str | None:\n The value of the `href` attribute, or None if absent."
|
||||||
|
},
|
||||||
|
"parse_table": {
|
||||||
|
"name": "parse_table",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.parser.HTMLParser.parse_table",
|
||||||
|
"signature": "parse_table(table: Tag) -> list[list[str]]",
|
||||||
|
"docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table (Tag):\n BeautifulSoup tag representing a `<table>`.\n\nReturns:\n list[list[str]]:\n A list of rows, where each row is a list of cell text values."
|
||||||
|
},
|
||||||
|
"parse_meta": {
|
||||||
|
"name": "parse_meta",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.parser.HTMLParser.parse_meta",
|
||||||
|
"signature": "parse_meta() -> dict[str, Any]",
|
||||||
|
"docstring": "Extract high-level metadata from the HTML document.\n\nReturns:\n dict[str, Any]:\n Dictionary containing extracted metadata.\n\nNotes:\n **Responsibilities:**\n\n - Extract high-level metadata from the HTML document.\n - This includes: Document title, `<meta>` tag name/property to\n content mappings."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
143
docs/mcp/modules/omniread.html.scraper.json
Normal file
143
docs/mcp/modules/omniread.html.scraper.json
Normal file
@@ -0,0 +1,143 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.html.scraper",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.html.scraper",
|
||||||
|
"docstring": "# Summary\n\nHTML scraping implementation for OmniRead.\n\nThis module provides an HTTP-based scraper for retrieving HTML documents.\nIt implements the core `BaseScraper` contract using `httpx` as the transport\nlayer.\n\nThis scraper is responsible for:\n\n- Fetching raw HTML bytes over HTTP(S)\n- Validating response content type\n- Attaching HTTP metadata to the returned content\n\nThis scraper is not responsible for:\n\n- Parsing or interpreting HTML\n- Retrying failed requests\n- Managing crawl policies or rate limiting",
|
||||||
|
"objects": {
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.html.scraper.Content",
|
||||||
|
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.Content.raw",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.Content.source",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.Content.content_type",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.Content.metadata",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.html.scraper.ContentType",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.ContentType.HTML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.ContentType.PDF",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.ContentType.XLSX",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.ContentType.CSV",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.ContentType.JSON",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.ContentType.XML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseScraper": {
|
||||||
|
"name": "BaseScraper",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.html.scraper.BaseScraper",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.scraper.BaseScraper.fetch",
|
||||||
|
"signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None)",
|
||||||
|
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"HTMLScraper": {
|
||||||
|
"name": "HTMLScraper",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.html.scraper.HTMLScraper",
|
||||||
|
"signature": "HTMLScraper(*, client: httpx.Client | None = None, timeout: float = 15.0, headers: Mapping[str, str] | None = None, follow_redirects: bool = True)",
|
||||||
|
"docstring": "Base HTML scraper using `httpx`.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns\n them as raw content wrapped in a `Content` object.\n - Fetches raw bytes and metadata only.\n - The scraper uses `httpx.Client` for HTTP requests, enforces an\n HTML content type, and preserves HTTP response metadata.\n\n **Constraints:**\n\n - The scraper does not: Parse HTML, perform retries or backoff,\n handle non-HTML responses.",
|
||||||
|
"members": {
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.HTMLScraper.content_type",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"validate_content_type": {
|
||||||
|
"name": "validate_content_type",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.scraper.HTMLScraper.validate_content_type",
|
||||||
|
"signature": "validate_content_type(response: httpx.Response) -> None",
|
||||||
|
"docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content."
|
||||||
|
},
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.html.scraper.HTMLScraper.fetch",
|
||||||
|
"signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None) -> Content",
|
||||||
|
"docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
2699
docs/mcp/modules/omniread.json
Normal file
2699
docs/mcp/modules/omniread.json
Normal file
File diff suppressed because it is too large
Load Diff
41
docs/mcp/modules/omniread.pdf.client.json
Normal file
41
docs/mcp/modules/omniread.pdf.client.json
Normal file
@@ -0,0 +1,41 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.pdf.client",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.pdf.client",
|
||||||
|
"docstring": "# Summary\n\nPDF client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw PDF\nbytes from a concrete backing store.\n\nClients provide low-level access to PDF binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems",
|
||||||
|
"objects": {
|
||||||
|
"BasePDFClient": {
|
||||||
|
"name": "BasePDFClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.pdf.client.BasePDFClient",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Abstract client responsible for retrieving PDF bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full PDF binary payload.\n - Raise retrieval-specific errors on failure.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.pdf.client.BasePDFClient.fetch",
|
||||||
|
"signature": "fetch(source: Any) -> bytes",
|
||||||
|
"docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF location, such as a file path, object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"FileSystemPDFClient": {
|
||||||
|
"name": "FileSystemPDFClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.pdf.client.FileSystemPDFClient",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "PDF client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads PDF files directly from the disk and returns\n their raw binary contents.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.pdf.client.FileSystemPDFClient.fetch",
|
||||||
|
"signature": "fetch(path: Path) -> bytes",
|
||||||
|
"docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the PDF file.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
377
docs/mcp/modules/omniread.pdf.json
Normal file
377
docs/mcp/modules/omniread.pdf.json
Normal file
@@ -0,0 +1,377 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.pdf",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.pdf",
|
||||||
|
"docstring": "# Summary\n\nPDF format implementation for OmniRead.\n\nThis package provides **PDF-specific implementations** of the core OmniRead\ncontracts defined in `omniread.core`.\n\nUnlike HTML, PDF handling requires an explicit client layer for document\naccess. This package therefore includes:\n\n- PDF clients for acquiring raw PDF data.\n- PDF scrapers that coordinate client access.\n- PDF parsers that extract structured content from PDF binaries.\n\nPublic exports from this package represent the supported PDF pipeline\nand are safe for consumers to import directly when working with PDFs.\n\n---\n\n# Public API\n\n- `FileSystemPDFClient`\n- `PDFScraper`\n- `PDFParser`\n\n---",
|
||||||
|
"objects": {
|
||||||
|
"FileSystemPDFClient": {
|
||||||
|
"name": "FileSystemPDFClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.pdf.FileSystemPDFClient",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "PDF client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads PDF files directly from the disk and returns\n their raw binary contents.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.pdf.FileSystemPDFClient.fetch",
|
||||||
|
"signature": "fetch(path: Path)",
|
||||||
|
"docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the PDF file.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"PDFScraper": {
|
||||||
|
"name": "PDFScraper",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.pdf.PDFScraper",
|
||||||
|
"signature": "PDFScraper(*, client: BasePDFClient)",
|
||||||
|
"docstring": "Scraper for PDF sources.\n\nNotes:\n **Responsibilities:**\n\n - Delegates byte retrieval to a PDF client and normalizes output\n into `Content`.\n - Preserves caller-provided metadata.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.pdf.PDFScraper.fetch",
|
||||||
|
"signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None)",
|
||||||
|
"docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"PDFParser": {
|
||||||
|
"name": "PDFParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.pdf.PDFParser",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Base PDF parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces PDF content-type compatibility and provides\n the extension point for implementing concrete PDF parsing strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.PDFParser.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser (PDF only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.pdf.PDFParser.parse",
|
||||||
|
"signature": "parse()",
|
||||||
|
"docstring": "Parse PDF content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the PDF binary payload and\n return a deterministic, structured output."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"client": {
|
||||||
|
"name": "client",
|
||||||
|
"kind": "module",
|
||||||
|
"path": "omniread.pdf.client",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "# Summary\n\nPDF client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw PDF\nbytes from a concrete backing store.\n\nClients provide low-level access to PDF binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems",
|
||||||
|
"members": {
|
||||||
|
"BasePDFClient": {
|
||||||
|
"name": "BasePDFClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.pdf.client.BasePDFClient",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Abstract client responsible for retrieving PDF bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full PDF binary payload.\n - Raise retrieval-specific errors on failure.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.pdf.client.BasePDFClient.fetch",
|
||||||
|
"signature": "fetch(source: Any) -> bytes",
|
||||||
|
"docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF location, such as a file path, object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"FileSystemPDFClient": {
|
||||||
|
"name": "FileSystemPDFClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.pdf.client.FileSystemPDFClient",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "PDF client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads PDF files directly from the disk and returns\n their raw binary contents.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.pdf.client.FileSystemPDFClient.fetch",
|
||||||
|
"signature": "fetch(path: Path) -> bytes",
|
||||||
|
"docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the PDF file.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"parser": {
|
||||||
|
"name": "parser",
|
||||||
|
"kind": "module",
|
||||||
|
"path": "omniread.pdf.parser",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "# Summary\n\nPDF parser base implementations for OmniRead.\n\nThis module defines the **PDF-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for PDF content.\n\nPDF parsers are responsible for interpreting binary PDF data and producing\nstructured representations suitable for downstream consumption.",
|
||||||
|
"members": {
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.pdf.parser.ContentType",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.parser.ContentType.HTML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.parser.ContentType.PDF",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.parser.ContentType.XLSX",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.parser.ContentType.CSV",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.parser.ContentType.JSON",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.parser.ContentType.XML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseParser": {
|
||||||
|
"name": "BaseParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.pdf.parser.BaseParser",
|
||||||
|
"signature": "BaseParser(content: Content)",
|
||||||
|
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.parser.BaseParser.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic."
|
||||||
|
},
|
||||||
|
"content": {
|
||||||
|
"name": "content",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.parser.BaseParser.content",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.pdf.parser.BaseParser.parse",
|
||||||
|
"signature": "parse()",
|
||||||
|
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
|
||||||
|
},
|
||||||
|
"supports": {
|
||||||
|
"name": "supports",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.pdf.parser.BaseParser.supports",
|
||||||
|
"signature": "supports()",
|
||||||
|
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"T": {
|
||||||
|
"name": "T",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.parser.T",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"PDFParser": {
|
||||||
|
"name": "PDFParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.pdf.parser.PDFParser",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Base PDF parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces PDF content-type compatibility and provides\n the extension point for implementing concrete PDF parsing strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.parser.PDFParser.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser (PDF only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.pdf.parser.PDFParser.parse",
|
||||||
|
"signature": "parse() -> T",
|
||||||
|
"docstring": "Parse PDF content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the PDF binary payload and\n return a deterministic, structured output."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"scraper": {
|
||||||
|
"name": "scraper",
|
||||||
|
"kind": "module",
|
||||||
|
"path": "omniread.pdf.scraper",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "# Summary\n\nPDF scraping implementation for OmniRead.\n\nThis module provides a PDF-specific scraper that coordinates PDF byte\nretrieval via a client and normalizes the result into a `Content` object.\n\nThe scraper implements the core `BaseScraper` contract while delegating\nall storage and access concerns to a `BasePDFClient` implementation.",
|
||||||
|
"members": {
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.pdf.scraper.Content",
|
||||||
|
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.scraper.Content.raw",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.scraper.Content.source",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.scraper.Content.content_type",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.scraper.Content.metadata",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.pdf.scraper.ContentType",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.scraper.ContentType.HTML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.scraper.ContentType.PDF",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.scraper.ContentType.XLSX",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.scraper.ContentType.CSV",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.scraper.ContentType.JSON",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.scraper.ContentType.XML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseScraper": {
|
||||||
|
"name": "BaseScraper",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.pdf.scraper.BaseScraper",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.pdf.scraper.BaseScraper.fetch",
|
||||||
|
"signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None)",
|
||||||
|
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BasePDFClient": {
|
||||||
|
"name": "BasePDFClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.pdf.scraper.BasePDFClient",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Abstract client responsible for retrieving PDF bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full PDF binary payload.\n - Raise retrieval-specific errors on failure.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.pdf.scraper.BasePDFClient.fetch",
|
||||||
|
"signature": "fetch(source: Any)",
|
||||||
|
"docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF location, such as a file path, object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"PDFScraper": {
|
||||||
|
"name": "PDFScraper",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.pdf.scraper.PDFScraper",
|
||||||
|
"signature": "PDFScraper(*, client: BasePDFClient)",
|
||||||
|
"docstring": "Scraper for PDF sources.\n\nNotes:\n **Responsibilities:**\n\n - Delegates byte retrieval to a PDF client and normalizes output\n into `Content`.\n - Preserves caller-provided metadata.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.pdf.scraper.PDFScraper.fetch",
|
||||||
|
"signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None) -> Content",
|
||||||
|
"docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
127
docs/mcp/modules/omniread.pdf.parser.json
Normal file
127
docs/mcp/modules/omniread.pdf.parser.json
Normal file
@@ -0,0 +1,127 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.pdf.parser",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.pdf.parser",
|
||||||
|
"docstring": "# Summary\n\nPDF parser base implementations for OmniRead.\n\nThis module defines the **PDF-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for PDF content.\n\nPDF parsers are responsible for interpreting binary PDF data and producing\nstructured representations suitable for downstream consumption.",
|
||||||
|
"objects": {
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.pdf.parser.ContentType",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.parser.ContentType.HTML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.parser.ContentType.PDF",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.parser.ContentType.XLSX",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.parser.ContentType.CSV",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.parser.ContentType.JSON",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.parser.ContentType.XML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseParser": {
|
||||||
|
"name": "BaseParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.pdf.parser.BaseParser",
|
||||||
|
"signature": "BaseParser(content: Content)",
|
||||||
|
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.parser.BaseParser.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic."
|
||||||
|
},
|
||||||
|
"content": {
|
||||||
|
"name": "content",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.parser.BaseParser.content",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.pdf.parser.BaseParser.parse",
|
||||||
|
"signature": "parse()",
|
||||||
|
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
|
||||||
|
},
|
||||||
|
"supports": {
|
||||||
|
"name": "supports",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.pdf.parser.BaseParser.supports",
|
||||||
|
"signature": "supports()",
|
||||||
|
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"T": {
|
||||||
|
"name": "T",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.parser.T",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"PDFParser": {
|
||||||
|
"name": "PDFParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.pdf.parser.PDFParser",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Base PDF parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces PDF content-type compatibility and provides\n the extension point for implementing concrete PDF parsing strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.parser.PDFParser.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser (PDF only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.pdf.parser.PDFParser.parse",
|
||||||
|
"signature": "parse() -> T",
|
||||||
|
"docstring": "Parse PDF content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the PDF binary payload and\n return a deterministic, structured output."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
145
docs/mcp/modules/omniread.pdf.scraper.json
Normal file
145
docs/mcp/modules/omniread.pdf.scraper.json
Normal file
@@ -0,0 +1,145 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.pdf.scraper",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.pdf.scraper",
|
||||||
|
"docstring": "# Summary\n\nPDF scraping implementation for OmniRead.\n\nThis module provides a PDF-specific scraper that coordinates PDF byte\nretrieval via a client and normalizes the result into a `Content` object.\n\nThe scraper implements the core `BaseScraper` contract while delegating\nall storage and access concerns to a `BasePDFClient` implementation.",
|
||||||
|
"objects": {
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.pdf.scraper.Content",
|
||||||
|
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.scraper.Content.raw",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.scraper.Content.source",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.scraper.Content.content_type",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.scraper.Content.metadata",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.pdf.scraper.ContentType",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.scraper.ContentType.HTML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.scraper.ContentType.PDF",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.scraper.ContentType.XLSX",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.scraper.ContentType.CSV",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.scraper.ContentType.JSON",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.scraper.ContentType.XML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseScraper": {
|
||||||
|
"name": "BaseScraper",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.pdf.scraper.BaseScraper",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.pdf.scraper.BaseScraper.fetch",
|
||||||
|
"signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None)",
|
||||||
|
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BasePDFClient": {
|
||||||
|
"name": "BasePDFClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.pdf.scraper.BasePDFClient",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Abstract client responsible for retrieving PDF bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full PDF binary payload.\n - Raise retrieval-specific errors on failure.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.pdf.scraper.BasePDFClient.fetch",
|
||||||
|
"signature": "fetch(source: Any)",
|
||||||
|
"docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF location, such as a file path, object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"PDFScraper": {
|
||||||
|
"name": "PDFScraper",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.pdf.scraper.PDFScraper",
|
||||||
|
"signature": "PDFScraper(*, client: BasePDFClient)",
|
||||||
|
"docstring": "Scraper for PDF sources.\n\nNotes:\n **Responsibilities:**\n\n - Delegates byte retrieval to a PDF client and normalizes output\n into `Content`.\n - Preserves caller-provided metadata.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.pdf.scraper.PDFScraper.fetch",
|
||||||
|
"signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None) -> Content",
|
||||||
|
"docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
41
docs/mcp/modules/omniread.xlsx.client.json
Normal file
41
docs/mcp/modules/omniread.xlsx.client.json
Normal file
@@ -0,0 +1,41 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.xlsx.client",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.xlsx.client",
|
||||||
|
"docstring": "# Summary\n\nXLSX client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw\nOffice Open XML spreadsheet bytes from a concrete backing store.\n\nClients provide low-level access to xlsx binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems",
|
||||||
|
"objects": {
|
||||||
|
"BaseXlsxClient": {
|
||||||
|
"name": "BaseXlsxClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.client.BaseXlsxClient",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Abstract client responsible for retrieving spreadsheet bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full xlsx binary payload.\n - Raise retrieval-specific errors on failure.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.client.BaseXlsxClient.fetch",
|
||||||
|
"signature": "fetch(source: Any) -> bytes",
|
||||||
|
"docstring": "Fetch raw xlsx bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"FileSystemXlsxClient": {
|
||||||
|
"name": "FileSystemXlsxClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.client.FileSystemXlsxClient",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "XLSX client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads spreadsheet files directly from the disk and\n returns their raw binary contents.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.client.FileSystemXlsxClient.fetch",
|
||||||
|
"signature": "fetch(path: Path) -> bytes",
|
||||||
|
"docstring": "Read an xlsx file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the spreadsheet file.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
520
docs/mcp/modules/omniread.xlsx.json
Normal file
520
docs/mcp/modules/omniread.xlsx.json
Normal file
@@ -0,0 +1,520 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.xlsx",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.xlsx",
|
||||||
|
"docstring": "# Summary\n\nXLSX subpackage for OmniRead.\n\nProvides acquisition and parsing of Office Open XML spreadsheet (xlsx)\ncontent:\n\n- `BaseXlsxClient`: abstract backing-store client for xlsx bytes.\n- `FileSystemXlsxClient`: local filesystem implementation.\n- `XlsxScraper`: wraps fetched bytes into canonical `Content`.\n- `XlsxParserBase`: content-type-enforcing parser contract.\n- `XlsxParser`: generic string-row parser built on openpyxl.",
|
||||||
|
"objects": {
|
||||||
|
"BaseXlsxClient": {
|
||||||
|
"name": "BaseXlsxClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.BaseXlsxClient",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Abstract client responsible for retrieving spreadsheet bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full xlsx binary payload.\n - Raise retrieval-specific errors on failure.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.BaseXlsxClient.fetch",
|
||||||
|
"signature": "fetch(source: Any)",
|
||||||
|
"docstring": "Fetch raw xlsx bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"FileSystemXlsxClient": {
|
||||||
|
"name": "FileSystemXlsxClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.FileSystemXlsxClient",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "XLSX client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads spreadsheet files directly from the disk and\n returns their raw binary contents.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.FileSystemXlsxClient.fetch",
|
||||||
|
"signature": "fetch(path: Path)",
|
||||||
|
"docstring": "Read an xlsx file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the spreadsheet file.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"XlsxParser": {
|
||||||
|
"name": "XlsxParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.XlsxParser",
|
||||||
|
"signature": "XlsxParser(content: Content, *, data_only: bool = True, read_only: bool = True)",
|
||||||
|
"docstring": "Generic xlsx parser producing string rows from a worksheet.\n\nNotes:\n **Responsibilities:**\n\n - Lazily load the workbook owned by the parser's content.\n - Normalize cells (including dates and numeric values) into\n deterministic string representations.\n - Expose sheet discovery and row extraction helpers.\n\n **Constraints:**\n\n - Cells are rendered with ``str(value)`` after trimming; date and\n datetime values are rendered in ISO format. Consumers requiring\n locale-specific formatting must convert on their side.",
|
||||||
|
"members": {
|
||||||
|
"workbook": {
|
||||||
|
"name": "workbook",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.XlsxParser.workbook",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "The lazily loaded workbook backing this parser's content."
|
||||||
|
},
|
||||||
|
"sheet_names": {
|
||||||
|
"name": "sheet_names",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.XlsxParser.sheet_names",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Names of all worksheets contained in the workbook."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.XlsxParser.parse",
|
||||||
|
"signature": "parse()",
|
||||||
|
"docstring": "Parse the first worksheet into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the default (first) worksheet."
|
||||||
|
},
|
||||||
|
"rows": {
|
||||||
|
"name": "rows",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.XlsxParser.rows",
|
||||||
|
"signature": "rows(sheet: int | str | None = None, *, skip_empty: bool = True)",
|
||||||
|
"docstring": "Extract normalized string rows from a worksheet.\n\nArgs:\n sheet (int | str | None):\n Worksheet index or title; defaults to the first worksheet.\n skip_empty (bool):\n When True (default), rows whose cells are all blank are omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row.\n\nRaises:\n ValueError:\n If the requested sheet does not exist."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"XlsxParserBase": {
|
||||||
|
"name": "XlsxParserBase",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.XlsxParserBase",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Base xlsx parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces xlsx content-type compatibility and provides\n the extension point for implementing concrete xlsx parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.XlsxParserBase.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser (XLSX only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.XlsxParserBase.parse",
|
||||||
|
"signature": "parse()",
|
||||||
|
"docstring": "Parse xlsx content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"XlsxScraper": {
|
||||||
|
"name": "XlsxScraper",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.XlsxScraper",
|
||||||
|
"signature": "XlsxScraper(*, client: BaseXlsxClient)",
|
||||||
|
"docstring": "Scraper for xlsx spreadsheet documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw xlsx bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n XLSX content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.XlsxScraper.fetch",
|
||||||
|
"signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None)",
|
||||||
|
"docstring": "Fetch an xlsx document from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw xlsx bytes, source\n identifier, XLSX content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"client": {
|
||||||
|
"name": "client",
|
||||||
|
"kind": "module",
|
||||||
|
"path": "omniread.xlsx.client",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "# Summary\n\nXLSX client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw\nOffice Open XML spreadsheet bytes from a concrete backing store.\n\nClients provide low-level access to xlsx binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems",
|
||||||
|
"members": {
|
||||||
|
"BaseXlsxClient": {
|
||||||
|
"name": "BaseXlsxClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.client.BaseXlsxClient",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Abstract client responsible for retrieving spreadsheet bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full xlsx binary payload.\n - Raise retrieval-specific errors on failure.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.client.BaseXlsxClient.fetch",
|
||||||
|
"signature": "fetch(source: Any) -> bytes",
|
||||||
|
"docstring": "Fetch raw xlsx bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"FileSystemXlsxClient": {
|
||||||
|
"name": "FileSystemXlsxClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.client.FileSystemXlsxClient",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "XLSX client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads spreadsheet files directly from the disk and\n returns their raw binary contents.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.client.FileSystemXlsxClient.fetch",
|
||||||
|
"signature": "fetch(path: Path) -> bytes",
|
||||||
|
"docstring": "Read an xlsx file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the spreadsheet file.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"parser": {
|
||||||
|
"name": "parser",
|
||||||
|
"kind": "module",
|
||||||
|
"path": "omniread.xlsx.parser",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "# Summary\n\nXLSX parser implementations for OmniRead.\n\nThis module provides a concrete, generic parser for Office Open XML\nspreadsheets. It exposes workbook sheets as lists of string rows so\ndownstream consumers can interpret tabular content without depending on\nopenpyxl directly.\n\nThe parser is intentionally statement-agnostic: it performs no header\ndetection or column interpretation beyond basic cell normalization.",
|
||||||
|
"members": {
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.parser.Content",
|
||||||
|
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.Content.raw",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.Content.source",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.Content.content_type",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.Content.metadata",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"XlsxParserBase": {
|
||||||
|
"name": "XlsxParserBase",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParserBase",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Base xlsx parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces xlsx content-type compatibility and provides\n the extension point for implementing concrete xlsx parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParserBase.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser (XLSX only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParserBase.parse",
|
||||||
|
"signature": "parse()",
|
||||||
|
"docstring": "Parse xlsx content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"XlsxParser": {
|
||||||
|
"name": "XlsxParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParser",
|
||||||
|
"signature": "XlsxParser(content: Content, *, data_only: bool = True, read_only: bool = True)",
|
||||||
|
"docstring": "Generic xlsx parser producing string rows from a worksheet.\n\nNotes:\n **Responsibilities:**\n\n - Lazily load the workbook owned by the parser's content.\n - Normalize cells (including dates and numeric values) into\n deterministic string representations.\n - Expose sheet discovery and row extraction helpers.\n\n **Constraints:**\n\n - Cells are rendered with ``str(value)`` after trimming; date and\n datetime values are rendered in ISO format. Consumers requiring\n locale-specific formatting must convert on their side.",
|
||||||
|
"members": {
|
||||||
|
"workbook": {
|
||||||
|
"name": "workbook",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParser.workbook",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "The lazily loaded workbook backing this parser's content."
|
||||||
|
},
|
||||||
|
"sheet_names": {
|
||||||
|
"name": "sheet_names",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParser.sheet_names",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Names of all worksheets contained in the workbook."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParser.parse",
|
||||||
|
"signature": "parse() -> list[list[str]]",
|
||||||
|
"docstring": "Parse the first worksheet into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the default (first) worksheet."
|
||||||
|
},
|
||||||
|
"rows": {
|
||||||
|
"name": "rows",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParser.rows",
|
||||||
|
"signature": "rows(sheet: int | str | None = None, *, skip_empty: bool = True) -> list[list[str]]",
|
||||||
|
"docstring": "Extract normalized string rows from a worksheet.\n\nArgs:\n sheet (int | str | None):\n Worksheet index or title; defaults to the first worksheet.\n skip_empty (bool):\n When True (default), rows whose cells are all blank are omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row.\n\nRaises:\n ValueError:\n If the requested sheet does not exist."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"parser_base": {
|
||||||
|
"name": "parser_base",
|
||||||
|
"kind": "module",
|
||||||
|
"path": "omniread.xlsx.parser_base",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "# Summary\n\nXLSX parser base implementation for OmniRead.\n\nThis module defines the **XLSX-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for Office Open\nXML spreadsheet content.",
|
||||||
|
"members": {
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType.HTML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType.PDF",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType.XLSX",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType.CSV",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType.JSON",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType.XML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseParser": {
|
||||||
|
"name": "BaseParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.parser_base.BaseParser",
|
||||||
|
"signature": "BaseParser(content: Content)",
|
||||||
|
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.BaseParser.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic."
|
||||||
|
},
|
||||||
|
"content": {
|
||||||
|
"name": "content",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.BaseParser.content",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.parser_base.BaseParser.parse",
|
||||||
|
"signature": "parse()",
|
||||||
|
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
|
||||||
|
},
|
||||||
|
"supports": {
|
||||||
|
"name": "supports",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.parser_base.BaseParser.supports",
|
||||||
|
"signature": "supports()",
|
||||||
|
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"T": {
|
||||||
|
"name": "T",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.T",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"XlsxParserBase": {
|
||||||
|
"name": "XlsxParserBase",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.parser_base.XlsxParserBase",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Base xlsx parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces xlsx content-type compatibility and provides\n the extension point for implementing concrete xlsx parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.XlsxParserBase.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser (XLSX only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.parser_base.XlsxParserBase.parse",
|
||||||
|
"signature": "parse() -> T",
|
||||||
|
"docstring": "Parse xlsx content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"scraper": {
|
||||||
|
"name": "scraper",
|
||||||
|
"kind": "module",
|
||||||
|
"path": "omniread.xlsx.scraper",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "# Summary\n\nXLSX scraper for OmniRead.\n\nThis module defines the scraper responsible for acquiring raw Office Open\nXML spreadsheet content from a backing store via a configured client.\n\nThe scraper does not interpret or parse the acquired bytes; it wraps them in\nthe canonical `Content` model.",
|
||||||
|
"members": {
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.scraper.Content",
|
||||||
|
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.Content.raw",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.Content.source",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.Content.content_type",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.Content.metadata",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType.HTML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType.PDF",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType.XLSX",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType.CSV",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType.JSON",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType.XML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseXlsxClient": {
|
||||||
|
"name": "BaseXlsxClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.scraper.BaseXlsxClient",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Abstract client responsible for retrieving spreadsheet bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full xlsx binary payload.\n - Raise retrieval-specific errors on failure.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.scraper.BaseXlsxClient.fetch",
|
||||||
|
"signature": "fetch(source: Any)",
|
||||||
|
"docstring": "Fetch raw xlsx bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"XlsxScraper": {
|
||||||
|
"name": "XlsxScraper",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.scraper.XlsxScraper",
|
||||||
|
"signature": "XlsxScraper(*, client: BaseXlsxClient)",
|
||||||
|
"docstring": "Scraper for xlsx spreadsheet documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw xlsx bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n XLSX content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.scraper.XlsxScraper.fetch",
|
||||||
|
"signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None) -> Content",
|
||||||
|
"docstring": "Fetch an xlsx document from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw xlsx bytes, source\n identifier, XLSX content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
106
docs/mcp/modules/omniread.xlsx.parser.json
Normal file
106
docs/mcp/modules/omniread.xlsx.parser.json
Normal file
@@ -0,0 +1,106 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.xlsx.parser",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.xlsx.parser",
|
||||||
|
"docstring": "# Summary\n\nXLSX parser implementations for OmniRead.\n\nThis module provides a concrete, generic parser for Office Open XML\nspreadsheets. It exposes workbook sheets as lists of string rows so\ndownstream consumers can interpret tabular content without depending on\nopenpyxl directly.\n\nThe parser is intentionally statement-agnostic: it performs no header\ndetection or column interpretation beyond basic cell normalization.",
|
||||||
|
"objects": {
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.parser.Content",
|
||||||
|
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.Content.raw",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.Content.source",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.Content.content_type",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.Content.metadata",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"XlsxParserBase": {
|
||||||
|
"name": "XlsxParserBase",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParserBase",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Base xlsx parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces xlsx content-type compatibility and provides\n the extension point for implementing concrete xlsx parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParserBase.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser (XLSX only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParserBase.parse",
|
||||||
|
"signature": "parse()",
|
||||||
|
"docstring": "Parse xlsx content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"XlsxParser": {
|
||||||
|
"name": "XlsxParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParser",
|
||||||
|
"signature": "XlsxParser(content: Content, *, data_only: bool = True, read_only: bool = True)",
|
||||||
|
"docstring": "Generic xlsx parser producing string rows from a worksheet.\n\nNotes:\n **Responsibilities:**\n\n - Lazily load the workbook owned by the parser's content.\n - Normalize cells (including dates and numeric values) into\n deterministic string representations.\n - Expose sheet discovery and row extraction helpers.\n\n **Constraints:**\n\n - Cells are rendered with ``str(value)`` after trimming; date and\n datetime values are rendered in ISO format. Consumers requiring\n locale-specific formatting must convert on their side.",
|
||||||
|
"members": {
|
||||||
|
"workbook": {
|
||||||
|
"name": "workbook",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParser.workbook",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "The lazily loaded workbook backing this parser's content."
|
||||||
|
},
|
||||||
|
"sheet_names": {
|
||||||
|
"name": "sheet_names",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParser.sheet_names",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Names of all worksheets contained in the workbook."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParser.parse",
|
||||||
|
"signature": "parse() -> list[list[str]]",
|
||||||
|
"docstring": "Parse the first worksheet into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the default (first) worksheet."
|
||||||
|
},
|
||||||
|
"rows": {
|
||||||
|
"name": "rows",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParser.rows",
|
||||||
|
"signature": "rows(sheet: int | str | None = None, *, skip_empty: bool = True) -> list[list[str]]",
|
||||||
|
"docstring": "Extract normalized string rows from a worksheet.\n\nArgs:\n sheet (int | str | None):\n Worksheet index or title; defaults to the first worksheet.\n skip_empty (bool):\n When True (default), rows whose cells are all blank are omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row.\n\nRaises:\n ValueError:\n If the requested sheet does not exist."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
127
docs/mcp/modules/omniread.xlsx.parser_base.json
Normal file
127
docs/mcp/modules/omniread.xlsx.parser_base.json
Normal file
@@ -0,0 +1,127 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.xlsx.parser_base",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.xlsx.parser_base",
|
||||||
|
"docstring": "# Summary\n\nXLSX parser base implementation for OmniRead.\n\nThis module defines the **XLSX-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for Office Open\nXML spreadsheet content.",
|
||||||
|
"objects": {
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType.HTML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType.PDF",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType.XLSX",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType.CSV",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType.JSON",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType.XML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseParser": {
|
||||||
|
"name": "BaseParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.parser_base.BaseParser",
|
||||||
|
"signature": "BaseParser(content: Content)",
|
||||||
|
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.BaseParser.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic."
|
||||||
|
},
|
||||||
|
"content": {
|
||||||
|
"name": "content",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.BaseParser.content",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.parser_base.BaseParser.parse",
|
||||||
|
"signature": "parse()",
|
||||||
|
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
|
||||||
|
},
|
||||||
|
"supports": {
|
||||||
|
"name": "supports",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.parser_base.BaseParser.supports",
|
||||||
|
"signature": "supports()",
|
||||||
|
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"T": {
|
||||||
|
"name": "T",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.T",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"XlsxParserBase": {
|
||||||
|
"name": "XlsxParserBase",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.parser_base.XlsxParserBase",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Base xlsx parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces xlsx content-type compatibility and provides\n the extension point for implementing concrete xlsx parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.XlsxParserBase.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser (XLSX only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.parser_base.XlsxParserBase.parse",
|
||||||
|
"signature": "parse() -> T",
|
||||||
|
"docstring": "Parse xlsx content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
129
docs/mcp/modules/omniread.xlsx.scraper.json
Normal file
129
docs/mcp/modules/omniread.xlsx.scraper.json
Normal file
@@ -0,0 +1,129 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.xlsx.scraper",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.xlsx.scraper",
|
||||||
|
"docstring": "# Summary\n\nXLSX scraper for OmniRead.\n\nThis module defines the scraper responsible for acquiring raw Office Open\nXML spreadsheet content from a backing store via a configured client.\n\nThe scraper does not interpret or parse the acquired bytes; it wraps them in\nthe canonical `Content` model.",
|
||||||
|
"objects": {
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.scraper.Content",
|
||||||
|
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.Content.raw",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.Content.source",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.Content.content_type",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.Content.metadata",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType.HTML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType.PDF",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType.XLSX",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType.CSV",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType.JSON",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType.XML",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseXlsxClient": {
|
||||||
|
"name": "BaseXlsxClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.scraper.BaseXlsxClient",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Abstract client responsible for retrieving spreadsheet bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full xlsx binary payload.\n - Raise retrieval-specific errors on failure.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.scraper.BaseXlsxClient.fetch",
|
||||||
|
"signature": "fetch(source: Any)",
|
||||||
|
"docstring": "Fetch raw xlsx bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"XlsxScraper": {
|
||||||
|
"name": "XlsxScraper",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.scraper.XlsxScraper",
|
||||||
|
"signature": "XlsxScraper(*, client: BaseXlsxClient)",
|
||||||
|
"docstring": "Scraper for xlsx spreadsheet documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw xlsx bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n XLSX content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.scraper.XlsxScraper.fetch",
|
||||||
|
"signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None) -> Content",
|
||||||
|
"docstring": "Fetch an xlsx document from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw xlsx bytes, source\n identifier, XLSX content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
90
docs/mcp/nav.json
Normal file
90
docs/mcp/nav.json
Normal file
@@ -0,0 +1,90 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"module": "omniread",
|
||||||
|
"resource": "docs://modules/omniread"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.core",
|
||||||
|
"resource": "docs://modules/omniread.core"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.core.content",
|
||||||
|
"resource": "docs://modules/omniread.core.content"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.core.parser",
|
||||||
|
"resource": "docs://modules/omniread.core.parser"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.core.scraper",
|
||||||
|
"resource": "docs://modules/omniread.core.scraper"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.csv",
|
||||||
|
"resource": "docs://modules/omniread.csv"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.csv.client",
|
||||||
|
"resource": "docs://modules/omniread.csv.client"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.csv.parser",
|
||||||
|
"resource": "docs://modules/omniread.csv.parser"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.csv.parser_base",
|
||||||
|
"resource": "docs://modules/omniread.csv.parser_base"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.csv.scraper",
|
||||||
|
"resource": "docs://modules/omniread.csv.scraper"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.html",
|
||||||
|
"resource": "docs://modules/omniread.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.html.parser",
|
||||||
|
"resource": "docs://modules/omniread.html.parser"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.html.scraper",
|
||||||
|
"resource": "docs://modules/omniread.html.scraper"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.pdf",
|
||||||
|
"resource": "docs://modules/omniread.pdf"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.pdf.client",
|
||||||
|
"resource": "docs://modules/omniread.pdf.client"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.pdf.parser",
|
||||||
|
"resource": "docs://modules/omniread.pdf.parser"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.pdf.scraper",
|
||||||
|
"resource": "docs://modules/omniread.pdf.scraper"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.xlsx",
|
||||||
|
"resource": "docs://modules/omniread.xlsx"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.xlsx.client",
|
||||||
|
"resource": "docs://modules/omniread.xlsx.client"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.xlsx.parser",
|
||||||
|
"resource": "docs://modules/omniread.xlsx.parser"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.xlsx.parser_base",
|
||||||
|
"resource": "docs://modules/omniread.xlsx.parser_base"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.xlsx.scraper",
|
||||||
|
"resource": "docs://modules/omniread.xlsx.scraper"
|
||||||
|
}
|
||||||
|
]
|
||||||
99
docs/mkdocs.lib.yml
Normal file
99
docs/mkdocs.lib.yml
Normal file
@@ -0,0 +1,99 @@
|
|||||||
|
theme:
|
||||||
|
name: material
|
||||||
|
palette:
|
||||||
|
- scheme: slate
|
||||||
|
primary: deep purple
|
||||||
|
accent: cyan
|
||||||
|
font:
|
||||||
|
text: Inter
|
||||||
|
code: JetBrains Mono
|
||||||
|
features:
|
||||||
|
- navigation.sections
|
||||||
|
- navigation.expand
|
||||||
|
- navigation.top
|
||||||
|
- navigation.instant
|
||||||
|
- navigation.tracking
|
||||||
|
- navigation.indexes
|
||||||
|
- content.code.copy
|
||||||
|
- content.code.annotate
|
||||||
|
- content.tabs.link
|
||||||
|
- content.action.edit
|
||||||
|
- search.highlight
|
||||||
|
- search.share
|
||||||
|
- search.suggest
|
||||||
|
icon:
|
||||||
|
logo: material/book-open-page-variant
|
||||||
|
repo: fontawesome/brands/github
|
||||||
|
markdown_extensions:
|
||||||
|
- pymdownx.superfences
|
||||||
|
- pymdownx.inlinehilite
|
||||||
|
- pymdownx.snippets
|
||||||
|
- admonition
|
||||||
|
- pymdownx.details
|
||||||
|
- pymdownx.highlight:
|
||||||
|
linenums: true
|
||||||
|
anchor_linenums: true
|
||||||
|
line_spans: __span
|
||||||
|
pygments_lang_class: true
|
||||||
|
- pymdownx.tabbed:
|
||||||
|
alternate_style: true
|
||||||
|
- pymdownx.tasklist:
|
||||||
|
custom_checkbox: true
|
||||||
|
- tables
|
||||||
|
- footnotes
|
||||||
|
- pymdownx.caret
|
||||||
|
- pymdownx.tilde
|
||||||
|
- pymdownx.mark
|
||||||
|
extra_css:
|
||||||
|
- https://unpkg.com/dracula-prism/dist/css/dracula-prism.css
|
||||||
|
plugins:
|
||||||
|
- search
|
||||||
|
- mkdocstrings:
|
||||||
|
handlers:
|
||||||
|
python:
|
||||||
|
paths:
|
||||||
|
- ..
|
||||||
|
options:
|
||||||
|
docstring_style: google
|
||||||
|
show_source: false
|
||||||
|
show_signature_annotations: true
|
||||||
|
separate_signature: true
|
||||||
|
merge_init_into_class: true
|
||||||
|
inherited_members: true
|
||||||
|
annotations_path: brief
|
||||||
|
show_root_heading: true
|
||||||
|
group_by_category: true
|
||||||
|
show_category_heading: true
|
||||||
|
show_object_full_path: false
|
||||||
|
show_symbol_type_heading: true
|
||||||
|
site_name: omniread
|
||||||
|
docs_dir: lib
|
||||||
|
site_dir: ../site/lib
|
||||||
|
nav:
|
||||||
|
- Home: index.md
|
||||||
|
- Core API:
|
||||||
|
- core/index.md
|
||||||
|
- core/content.md
|
||||||
|
- core/parser.md
|
||||||
|
- core/scraper.md
|
||||||
|
- HTML Handling:
|
||||||
|
- html/index.md
|
||||||
|
- html/parser.md
|
||||||
|
- html/scraper.md
|
||||||
|
- PDF Handling:
|
||||||
|
- pdf/index.md
|
||||||
|
- pdf/client.md
|
||||||
|
- pdf/parser.md
|
||||||
|
- pdf/scraper.md
|
||||||
|
- CSV Handling:
|
||||||
|
- csv/index.md
|
||||||
|
- csv/client.md
|
||||||
|
- csv/parser_base.md
|
||||||
|
- csv/parser.md
|
||||||
|
- csv/scraper.md
|
||||||
|
- XLSX Handling:
|
||||||
|
- xlsx/index.md
|
||||||
|
- xlsx/client.md
|
||||||
|
- xlsx/parser_base.md
|
||||||
|
- xlsx/parser.md
|
||||||
|
- xlsx/scraper.md
|
||||||
69
docs/mkdocs.wiki.yml
Normal file
69
docs/mkdocs.wiki.yml
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
site_name: OmniRead Documentation
|
||||||
|
docs_dir: wiki
|
||||||
|
site_dir: ../site/wiki
|
||||||
|
|
||||||
|
nav:
|
||||||
|
- Home: index.md
|
||||||
|
- Overview: 01_overview.md
|
||||||
|
- How to Use: 02_how_to_use.md
|
||||||
|
- Extending OmniRead: 03_extending.md
|
||||||
|
- Development: 04_development.md
|
||||||
|
|
||||||
|
theme:
|
||||||
|
name: material
|
||||||
|
palette:
|
||||||
|
scheme: slate
|
||||||
|
primary: blue grey
|
||||||
|
accent: teal
|
||||||
|
font:
|
||||||
|
text: Roboto
|
||||||
|
code: JetBrains Mono
|
||||||
|
features:
|
||||||
|
- navigation.sections
|
||||||
|
- navigation.expand
|
||||||
|
- navigation.top
|
||||||
|
- navigation.instant
|
||||||
|
- navigation.tracking
|
||||||
|
- navigation.indexes
|
||||||
|
- content.code.copy
|
||||||
|
- content.code.annotate
|
||||||
|
- content.tabs.link
|
||||||
|
- content.action.edit
|
||||||
|
- search.highlight
|
||||||
|
- search.share
|
||||||
|
- search.suggest
|
||||||
|
- navigation.tabs
|
||||||
|
- toc.integrate
|
||||||
|
- header.autohide
|
||||||
|
- announce.dismiss
|
||||||
|
- footer.social
|
||||||
|
- content.code.select
|
||||||
|
- content.code.line_numbers
|
||||||
|
- content.tooltips
|
||||||
|
icon:
|
||||||
|
logo: material/book-open-page-variant
|
||||||
|
repo: fontawesome/brands/github
|
||||||
|
markdown_extensions:
|
||||||
|
- pymdownx.superfences
|
||||||
|
- pymdownx.inlinehilite
|
||||||
|
- pymdownx.snippets
|
||||||
|
- admonition
|
||||||
|
- pymdownx.details
|
||||||
|
- pymdownx.highlight:
|
||||||
|
linenums: true
|
||||||
|
anchor_linenums: true
|
||||||
|
line_spans: __span
|
||||||
|
pygments_lang_class: true
|
||||||
|
- pymdownx.tabbed:
|
||||||
|
alternate_style: true
|
||||||
|
- pymdownx.tasklist:
|
||||||
|
custom_checkbox: true
|
||||||
|
- tables
|
||||||
|
- footnotes
|
||||||
|
- pymdownx.caret
|
||||||
|
- pymdownx.tilde
|
||||||
|
- pymdownx.mark
|
||||||
|
extra_css:
|
||||||
|
- https://unpkg.com/dracula-prism/dist/css/dracula-prism.css
|
||||||
|
plugins:
|
||||||
|
- search
|
||||||
86
docs/wiki/01_overview.md
Normal file
86
docs/wiki/01_overview.md
Normal file
@@ -0,0 +1,86 @@
|
|||||||
|
# 🧱 Overview
|
||||||
|
|
||||||
|
OmniRead is designed as a **decoupled content engine** with three distinct
|
||||||
|
layers. Understanding them is the key to using and extending the library.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🏗️ Architecture
|
||||||
|
|
||||||
|
```text
|
||||||
|
┌─────────────────────────────┐
|
||||||
|
│ Source (URL, file, storage) │
|
||||||
|
└────────────┬────────────────┘
|
||||||
|
│
|
||||||
|
┌─────────▼──────────┐
|
||||||
|
│ Scraper / Client │ fetches raw bytes
|
||||||
|
└─────────┬──────────┘
|
||||||
|
│ returns
|
||||||
|
┌─────────▼──────────┐
|
||||||
|
│ Content │ raw + source + type
|
||||||
|
└─────────┬──────────┘
|
||||||
|
│
|
||||||
|
┌─────────▼──────────┐
|
||||||
|
│ Parser │ parse() → structured T
|
||||||
|
└────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
1. **Scraper** (`BaseScraper`) — fetches raw bytes from a source (HTTP URL,
|
||||||
|
filesystem path, object storage). Returns a `Content` instance.
|
||||||
|
Scrapers never interpret content.
|
||||||
|
2. **Content** (`Content`) — the canonical exchange model: raw bytes,
|
||||||
|
source identifier, and optional `ContentType` enum.
|
||||||
|
3. **Parser** (`BaseParser[T]`) — receives a `Content` and returns a
|
||||||
|
structured result of type `T` via `parse()`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 📦 The `Content` model
|
||||||
|
|
||||||
|
Defined in `omniread.core.content`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from omniread import Content, ContentType
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class Content:
|
||||||
|
raw: bytes
|
||||||
|
source: str
|
||||||
|
content_type: ContentType | None = None
|
||||||
|
```
|
||||||
|
|
||||||
|
- `raw` — the raw bytes exactly as retrieved.
|
||||||
|
- `source` — URL, file path, or logical name identifying the origin.
|
||||||
|
- `content_type` — optional `ContentType` enum value.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🎭 The `ContentType` enum
|
||||||
|
|
||||||
|
| Value | MIME | Used by |
|
||||||
|
|---|---|---|
|
||||||
|
| `HTML` | `text/html` | HTMLScraper |
|
||||||
|
| `PDF` | `application/pdf` | PDFScraper |
|
||||||
|
| `XLSX` | `application/vnd.openxmlformats-...` | XlsxScraper |
|
||||||
|
| `CSV` | `text/csv` | CsvScraper |
|
||||||
|
| `JSON` | `application/json` | — |
|
||||||
|
| `XML` | `application/xml` | — |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🧩 Format modules at a glance
|
||||||
|
|
||||||
|
| Module | Scraper | Parser | Client | Notes |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| `omniread.html` | `HTMLScraper` | `HTMLParser` | — | httpx + BeautifulSoup |
|
||||||
|
| `omniread.pdf` | `PDFScraper` | `PDFParser` | `FileSystemPDFClient` | explicit client layer |
|
||||||
|
| `omniread.csv` | `CsvScraper` | `CsvParser` | `FileSystemCsvClient` | stdlib csv module |
|
||||||
|
| `omniread.xlsx` | `XlsxScraper` | `XlsxParser` | `FileSystemXlsxClient` | openpyxl-backed |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 📚 Read Next
|
||||||
|
|
||||||
|
- [How to Use](02_how_to_use.md) — working examples per format.
|
||||||
|
- [Extending OmniRead](03_extending.md) — subclassing scrapers and parsers.
|
||||||
109
docs/wiki/02_how_to_use.md
Normal file
109
docs/wiki/02_how_to_use.md
Normal file
@@ -0,0 +1,109 @@
|
|||||||
|
# 🖥️ How to Use
|
||||||
|
|
||||||
|
Every format follows the same pipeline: **scrape → `Content` → parse**.
|
||||||
|
This page shows each supported format with working patterns.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🌐 HTML
|
||||||
|
|
||||||
|
```python
|
||||||
|
from omniread import HTMLScraper, HTMLParser
|
||||||
|
|
||||||
|
class TitleParser(HTMLParser[str]):
|
||||||
|
def parse(self) -> str:
|
||||||
|
return self._soup.title.string
|
||||||
|
|
||||||
|
scraper = HTMLScraper() # httpx under the hood
|
||||||
|
content = scraper.fetch("https://example.com")
|
||||||
|
|
||||||
|
title = TitleParser(content).parse()
|
||||||
|
```
|
||||||
|
|
||||||
|
`HTMLScraper` accepts an optional `client` (an `httpx.Client`) for transport
|
||||||
|
control — the test suite wires one to a mock transport.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 📕 PDF
|
||||||
|
|
||||||
|
PDFs need a **client** to supply raw bytes before parsing:
|
||||||
|
|
||||||
|
```python
|
||||||
|
from pathlib import Path
|
||||||
|
from omniread import FileSystemPDFClient, PDFScraper, PDFParser
|
||||||
|
|
||||||
|
class TextPDFParser(PDFParser[str]):
|
||||||
|
def parse(self) -> str:
|
||||||
|
# implement your extraction logic
|
||||||
|
return self.content.raw # bytes, decode as needed
|
||||||
|
|
||||||
|
client = FileSystemPDFClient()
|
||||||
|
scraper = PDFScraper(client=client)
|
||||||
|
content = scraper.fetch(Path("document.pdf"))
|
||||||
|
|
||||||
|
result = TextPDFParser(content).parse()
|
||||||
|
```
|
||||||
|
|
||||||
|
`PDFParser` subclasses receive `self.content` and implement `parse()`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 📊 CSV
|
||||||
|
|
||||||
|
```python
|
||||||
|
from omniread import FileSystemCsvClient, CsvScraper, CsvParser
|
||||||
|
|
||||||
|
scraper = CsvScraper(client=FileSystemCsvClient())
|
||||||
|
content = scraper.fetch("data.csv")
|
||||||
|
|
||||||
|
parser = CsvParser(content)
|
||||||
|
for row in parser.rows():
|
||||||
|
print(row)
|
||||||
|
```
|
||||||
|
|
||||||
|
`CsvParser.rows()` yields string rows trimmed of empties by default
|
||||||
|
(`skip_empty=True`).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 📑 XLSX
|
||||||
|
|
||||||
|
```python
|
||||||
|
from omniread import FileSystemXlsxClient, XlsxScraper, XlsxParser
|
||||||
|
|
||||||
|
scraper = XlsxScraper(client=FileSystemXlsxClient())
|
||||||
|
content = scraper.fetch("statement.xlsx")
|
||||||
|
|
||||||
|
parser = XlsxParser(content)
|
||||||
|
print(parser.sheet_names) # e.g. ["Statement"]
|
||||||
|
rows = parser.rows(sheet="Statement") # by name or index
|
||||||
|
all_rows = parser.parse() # alias for rows()
|
||||||
|
```
|
||||||
|
|
||||||
|
Key behaviors:
|
||||||
|
|
||||||
|
- `rows(skip_empty=False)` keeps blank rows (off by default).
|
||||||
|
- Cells render as trimmed strings; date cells convert to ISO format
|
||||||
|
(`2026-06-01T00:00:00`).
|
||||||
|
- `rows(sheet="Missing")` raises for an unknown sheet.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🔀 End-to-end flow
|
||||||
|
|
||||||
|
```python
|
||||||
|
content = scraper.fetch(source) # 1. acquire → Content
|
||||||
|
assert isinstance(content.raw, bytes)
|
||||||
|
assert content.content_type is not None
|
||||||
|
|
||||||
|
parser = MyParser(content) # 2. interpret → T
|
||||||
|
result = parser.parse()
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 📚 Read Next
|
||||||
|
|
||||||
|
- [Overview](01_overview.md) — the full architecture.
|
||||||
|
- [Extending OmniRead](03_extending.md) — custom clients and parsers.
|
||||||
104
docs/wiki/03_extending.md
Normal file
104
docs/wiki/03_extending.md
Normal file
@@ -0,0 +1,104 @@
|
|||||||
|
# 🧩 Extending OmniRead
|
||||||
|
|
||||||
|
OmniRead is meant to be extended by subclassing. All public extension points
|
||||||
|
are generic over their result type, so your parser returns exactly the shape
|
||||||
|
you need.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🧬 Custom parsers
|
||||||
|
|
||||||
|
Subclass `BaseParser[T]` (or a format parser) and implement `parse()`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
from pydantic import BaseModel
|
||||||
|
from omniread import HTMLParser
|
||||||
|
|
||||||
|
class Page(BaseModel):
|
||||||
|
title: str
|
||||||
|
content: str | None
|
||||||
|
|
||||||
|
class PageParser(HTMLParser[Page]):
|
||||||
|
def parse(self) -> Page:
|
||||||
|
soup = self._soup
|
||||||
|
div = soup.find("div", id="content")
|
||||||
|
return Page(
|
||||||
|
title=soup.title.string,
|
||||||
|
content=div.get_text() if div else None,
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
The parsed page is validated by Pydantic on construction — no manual
|
||||||
|
assertions required.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🧬 Custom PDF parsers
|
||||||
|
|
||||||
|
PDF binary layout is format-specific, so parsers return your own model:
|
||||||
|
|
||||||
|
```python
|
||||||
|
from typing import Literal
|
||||||
|
from pydantic import BaseModel
|
||||||
|
from omniread import PDFParser
|
||||||
|
|
||||||
|
class ParsedPDF(BaseModel):
|
||||||
|
size_bytes: int
|
||||||
|
magic: Literal[b"%PDF"]
|
||||||
|
|
||||||
|
class SimplePDFParser(PDFParser[ParsedPDF]):
|
||||||
|
def parse(self) -> ParsedPDF:
|
||||||
|
if not self.content.raw.startswith(b"%PDF"):
|
||||||
|
raise ValueError("Not a valid PDF")
|
||||||
|
return ParsedPDF(size_bytes=len(self.content.raw), magic=b"%PDF")
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🧬 Custom clients
|
||||||
|
|
||||||
|
Clients supply raw bytes to a scraper. For PDFs, subclass
|
||||||
|
`BasePDFClient` (or `FileSystemPDFClient`) and implement
|
||||||
|
`fetch(source) -> bytes`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
from omniread.pdf.client import BasePDFClient
|
||||||
|
|
||||||
|
class MockPDFClient(BasePDFClient):
|
||||||
|
def fetch(self, source):
|
||||||
|
return b"%PDF ..." # bytes for the logical identifier
|
||||||
|
```
|
||||||
|
|
||||||
|
The same pattern applies to `BaseCsvClient` and `BaseXlsxClient`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🚀 Custom scrapers
|
||||||
|
|
||||||
|
Festch something that a built-in scraper does not cover by extending
|
||||||
|
`BaseScraper`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
from omniread import BaseScraper, Content, ContentType
|
||||||
|
|
||||||
|
class StorageScraper(BaseScraper):
|
||||||
|
def fetch(self, source, *, metadata=None):
|
||||||
|
raw = my_object_storage.download(source) # your I/O
|
||||||
|
return Content(raw=raw, source=source, content_type=ContentType.JSON)
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## ✅ Extension checklist
|
||||||
|
|
||||||
|
1. Keep **scraper** and **parser** separate — never mix I/O into `parse()`.
|
||||||
|
2. Return `Content` from any scraper/client so downstream stays uniform.
|
||||||
|
3. Return a *typed* result from your parser (Pydantic model, dataclass, str).
|
||||||
|
4. Test your custom layers with a mock client, not a live network.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 📚 Read Next
|
||||||
|
|
||||||
|
- [How to Use](02_how_to_use.md) — built-in example flows.
|
||||||
|
- [Development](04_development.md) — running tests and docs.
|
||||||
87
docs/wiki/04_development.md
Normal file
87
docs/wiki/04_development.md
Normal file
@@ -0,0 +1,87 @@
|
|||||||
|
# 🛠️ Development
|
||||||
|
|
||||||
|
Working on `omniread` itself.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 📂 Repository layout
|
||||||
|
|
||||||
|
| Path | Purpose |
|
||||||
|
|---|---|
|
||||||
|
| `omniread/` | The library package (core + html, pdf, csv, xlsx modules) |
|
||||||
|
| `omniread/*.pyi` | Type stubs kept in sync with implementations |
|
||||||
|
| `tests/` | End-to-end and unit tests against mock transports/clients |
|
||||||
|
| `covers` | — |
|
||||||
|
| `docs/lib/` | Generated library reference (docforge, flat layout) |
|
||||||
|
| `docs/mcp/` | Machine-readable bundle served by the MCP server |
|
||||||
|
| `docs/wiki/` | This hand-written wiki |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🔧 Setup
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python -m venv .venv
|
||||||
|
.venv/Scripts/pip install -e ".[dev]"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🧪 Tests
|
||||||
|
|
||||||
|
Run the suite (offline; a mock httpx transport and mock PDF client are used):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
.venv/Scripts/pytest
|
||||||
|
```
|
||||||
|
|
||||||
|
Coverage spans end-to-end scrape → parse flows for HTML, PDF, and XLSX,
|
||||||
|
plus client validation and CSV/XLSX parsing edge cases.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## ✅ Quality gates
|
||||||
|
|
||||||
|
The CI quality gate runs, matching the Drone pipeline:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
.venv/Scripts/black --check .
|
||||||
|
.venv/Scripts/ruff check .
|
||||||
|
.venv/Scripts/mypy
|
||||||
|
.venv/Scripts/pytest
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 📝 Building documentation (docforge)
|
||||||
|
|
||||||
|
The site is generated by [`docforge`](https://git.aetoskia.com/aetos/doc-forge)
|
||||||
|
and served per kind under `site/{kind}`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
doc-forge build \
|
||||||
|
--mkdocs --mcp \
|
||||||
|
--module-is-source --module omniread \
|
||||||
|
--site-name "OmniRead"
|
||||||
|
```
|
||||||
|
|
||||||
|
- `--module-is-source` renders the flat `docs/lib/` layout (no nesting under
|
||||||
|
`omniread/`), matching `docforge.nav.yml` and `docs/mkdocs.lib.yml`.
|
||||||
|
- `--mcp` regenerates the structured bundle in `docs/mcp/`.
|
||||||
|
- `--wiki` builds this wiki.
|
||||||
|
- `gen_api.py` (if present) regenerates the API docs.
|
||||||
|
|
||||||
|
Preview locally:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
doc-forge serve --lib
|
||||||
|
doc-forge serve --wiki
|
||||||
|
doc-forge serve --mcp
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 📚 Read Next
|
||||||
|
|
||||||
|
- [Extending OmniRead](03_extending.md) — custom parsers, scrapers, clients.
|
||||||
|
- [Overview](01_overview.md) — the core architecture.
|
||||||
64
docs/wiki/index.md
Normal file
64
docs/wiki/index.md
Normal file
@@ -0,0 +1,64 @@
|
|||||||
|
# 📄 OmniRead — Format-Agnostic Content Acquisition and Parsing
|
||||||
|
|
||||||
|
`OmniRead` is a composable content ingestion framework with pluggable scrapers
|
||||||
|
and parsers for HTML, PDF, CSV, and XLSX. It cleanly separates fetching
|
||||||
|
(I/O) from interpretation (parsing) through a canonical `Content` exchange
|
||||||
|
model, giving each format its own scraper and parser while sharing one
|
||||||
|
uniform pipeline.
|
||||||
|
|
||||||
|
> **Doc model:** this wiki is written for humans — how‑to guides, examples,
|
||||||
|
> and extension recipes. The authoritative API contracts live in the code
|
||||||
|
> (GSDFC docstrings) and the machine‑readable bundle under `docs/mcp/`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🚀 Key Features
|
||||||
|
|
||||||
|
* 🧱 **Content/Scraper/Parser architecture** — fetch and interpret in
|
||||||
|
independent, testable layers
|
||||||
|
* 📄 **Canonical `Content` model** — raw bytes + metadata passed between all
|
||||||
|
layers via one dataclass
|
||||||
|
* 🌐 **HTML** — HTTP-based scraper with httpx; DOM parsing via BeautifulSoup
|
||||||
|
* 📕 **PDF** — client + scraper for raw PDF bytes; user-defined parsers
|
||||||
|
* 📊 **CSV** — standard-library CSV parser over `rows()` iterator
|
||||||
|
* 📑 **XLSX** — openpyxl-backed spreadsheet parser with sheet selection,
|
||||||
|
empty-row control, and ISO date conversion
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## ⚡ Quick Start
|
||||||
|
|
||||||
|
```python
|
||||||
|
from omniread import HTMLScraper, HTMLParser
|
||||||
|
|
||||||
|
class TitleParser(HTMLParser[str]):
|
||||||
|
def parse(self) -> str:
|
||||||
|
return self._soup.title.string
|
||||||
|
|
||||||
|
content = HTMLScraper().fetch("https://example.com")
|
||||||
|
title = TitleParser(content).parse()
|
||||||
|
assert title == "Example Domain"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 📁 Documentation Structure
|
||||||
|
|
||||||
|
| Section | What you'll find |
|
||||||
|
|---|---|
|
||||||
|
| [Overview](01_overview.md) | The three-layer architecture and the `Content` model |
|
||||||
|
| [How to Use](02_how_to_use.md) | Ready-to-run examples for HTML, PDF, CSV, and XLSX |
|
||||||
|
| [Extending OmniRead](03_extending.md) | Subclassing scrapers, parsers, and clients |
|
||||||
|
| [Development](04_development.md) | Setup, tests, and regenerating docs |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🔗 Related Resources
|
||||||
|
|
||||||
|
* **Source Code:** [Gitea Repository](https://git.aetoskia.com/aetos/omniread)
|
||||||
|
* **Internal PyPI:** [pip.aetoskia.com/simple/omniread](https://pip.aetoskia.com/simple/omniread)
|
||||||
|
* **CI:** Builds and publishes tagged releases, gated on black / ruff / mypy / pytest.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
© Aetoskia Internal — `omniread` 0.0.1
|
||||||
@@ -1,46 +0,0 @@
|
|||||||
"""
|
|
||||||
Programmatic MkDocs build script for OmniRead.
|
|
||||||
|
|
||||||
This script builds (or serves) the documentation by invoking MkDocs
|
|
||||||
*as a Python library*, not via shell commands.
|
|
||||||
|
|
||||||
Requirements:
|
|
||||||
- mkdocs
|
|
||||||
- mkdocs-material
|
|
||||||
- mkdocstrings[python]
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
python generate_docs.py
|
|
||||||
python generate_docs.py --serve
|
|
||||||
"""
|
|
||||||
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from mkdocs.commands import build as mkdocs_build
|
|
||||||
from mkdocs.commands import serve as mkdocs_serve
|
|
||||||
from mkdocs.config import load_config
|
|
||||||
|
|
||||||
|
|
||||||
PROJECT_ROOT = Path(__file__).resolve().parent
|
|
||||||
MKDOCS_YML = PROJECT_ROOT / "mkdocs.yml"
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
if not MKDOCS_YML.exists():
|
|
||||||
raise FileNotFoundError("mkdocs.yml not found at project root")
|
|
||||||
|
|
||||||
# Load MkDocs configuration programmatically
|
|
||||||
config = load_config(str(MKDOCS_YML))
|
|
||||||
|
|
||||||
# Decide mode
|
|
||||||
if "--serve" in sys.argv:
|
|
||||||
# Live-reload development server
|
|
||||||
mkdocs_serve.serve(config)
|
|
||||||
else:
|
|
||||||
# Static site build
|
|
||||||
mkdocs_build.build(config)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
53
mkdocs.yml
53
mkdocs.yml
@@ -1,53 +0,0 @@
|
|||||||
site_name: Aetoskia OmniRead
|
|
||||||
site_description: Format-agnostic document reading, parsing, and scraping framework
|
|
||||||
|
|
||||||
theme:
|
|
||||||
name: material
|
|
||||||
palette:
|
|
||||||
- scheme: slate
|
|
||||||
primary: deep purple
|
|
||||||
accent: cyan
|
|
||||||
font:
|
|
||||||
text: Inter
|
|
||||||
code: JetBrains Mono
|
|
||||||
features:
|
|
||||||
- navigation.tabs
|
|
||||||
- navigation.expand
|
|
||||||
- navigation.top
|
|
||||||
- navigation.instant
|
|
||||||
- content.code.copy
|
|
||||||
- content.code.annotate
|
|
||||||
|
|
||||||
plugins:
|
|
||||||
- search
|
|
||||||
- mkdocstrings:
|
|
||||||
handlers:
|
|
||||||
python:
|
|
||||||
paths: ["."]
|
|
||||||
options:
|
|
||||||
docstring_style: google
|
|
||||||
show_source: false
|
|
||||||
show_signature_annotations: true
|
|
||||||
separate_signature: true
|
|
||||||
merge_init_into_class: true
|
|
||||||
inherited_members: true
|
|
||||||
annotations_path: brief
|
|
||||||
show_root_heading: true
|
|
||||||
group_by_category: true
|
|
||||||
|
|
||||||
nav:
|
|
||||||
- Home: index.md
|
|
||||||
|
|
||||||
- Core (Contracts):
|
|
||||||
- Content Models: core/content.md
|
|
||||||
- Parsers: core/parser.md
|
|
||||||
- Scrapers: core/scraper.md
|
|
||||||
|
|
||||||
- HTML Implementation:
|
|
||||||
- HTML Parser: html/parser.md
|
|
||||||
- HTML Scraper: html/scraper.md
|
|
||||||
|
|
||||||
- PDF Implementation:
|
|
||||||
- PDF Client: pdf/client.md
|
|
||||||
- PDF Parser: pdf/parser.md
|
|
||||||
- PDF Scraper: pdf/scraper.md
|
|
||||||
@@ -1,48 +1,48 @@
|
|||||||
"""
|
"""
|
||||||
OmniRead — format-agnostic content acquisition and parsing framework.
|
# Summary
|
||||||
|
|
||||||
OmniRead provides a **cleanly layered architecture** for fetching, parsing,
|
`OmniRead` — format-agnostic content acquisition and parsing framework.
|
||||||
|
|
||||||
|
`OmniRead` provides a **cleanly layered architecture** for fetching, parsing,
|
||||||
and normalizing content from heterogeneous sources such as HTML documents
|
and normalizing content from heterogeneous sources such as HTML documents
|
||||||
and PDF files.
|
and PDF files.
|
||||||
|
|
||||||
The library is structured around three core concepts:
|
The library is structured around three core concepts:
|
||||||
|
|
||||||
1. **Content**
|
1. **`Content`**: A canonical, format-agnostic container representing raw content
|
||||||
A canonical, format-agnostic container representing raw content bytes
|
bytes and minimal contextual metadata.
|
||||||
and minimal contextual metadata.
|
2. **`Scrapers`**: Components responsible for *acquiring* raw content from a
|
||||||
|
source (HTTP, filesystem, object storage, etc.). `Scrapers` never interpret
|
||||||
2. **Scrapers**
|
|
||||||
Components responsible for *acquiring* raw content from a source
|
|
||||||
(HTTP, filesystem, object storage, etc.). Scrapers never interpret
|
|
||||||
content.
|
content.
|
||||||
|
3. **`Parsers`**: Components responsible for *interpreting* acquired content and
|
||||||
3. **Parsers**
|
|
||||||
Components responsible for *interpreting* acquired content and
|
|
||||||
converting it into structured, typed representations.
|
converting it into structured, typed representations.
|
||||||
|
|
||||||
OmniRead deliberately separates these responsibilities to ensure:
|
`OmniRead` deliberately separates these responsibilities to ensure:
|
||||||
- Clear boundaries between IO and interpretation
|
|
||||||
- Replaceable implementations per format
|
|
||||||
- Predictable, testable behavior
|
|
||||||
|
|
||||||
----------------------------------------------------------------------
|
- Clear boundaries between IO and interpretation.
|
||||||
Installation
|
- Replaceable implementations per format.
|
||||||
----------------------------------------------------------------------
|
- Predictable, testable behavior.
|
||||||
|
|
||||||
Install OmniRead using pip:
|
# Installation
|
||||||
|
|
||||||
|
Install `OmniRead` using pip:
|
||||||
|
|
||||||
|
```bash
|
||||||
pip install omniread
|
pip install omniread
|
||||||
|
```
|
||||||
|
|
||||||
Or with Poetry:
|
Install OmniRead using Poetry:
|
||||||
|
```bash
|
||||||
poetry add omniread
|
poetry add omniread
|
||||||
|
```
|
||||||
|
|
||||||
----------------------------------------------------------------------
|
---
|
||||||
Basic Usage
|
|
||||||
----------------------------------------------------------------------
|
|
||||||
|
|
||||||
|
## Quick start
|
||||||
|
|
||||||
|
Example:
|
||||||
HTML example:
|
HTML example:
|
||||||
|
```python
|
||||||
from omniread import HTMLScraper, HTMLParser
|
from omniread import HTMLScraper, HTMLParser
|
||||||
|
|
||||||
scraper = HTMLScraper()
|
scraper = HTMLScraper()
|
||||||
@@ -54,9 +54,10 @@ HTML example:
|
|||||||
|
|
||||||
parser = TitleParser(content)
|
parser = TitleParser(content)
|
||||||
title = parser.parse()
|
title = parser.parse()
|
||||||
|
```
|
||||||
|
|
||||||
PDF example:
|
PDF example:
|
||||||
|
```python
|
||||||
from omniread import FileSystemPDFClient, PDFScraper, PDFParser
|
from omniread import FileSystemPDFClient, PDFScraper, PDFParser
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -71,46 +72,83 @@ PDF example:
|
|||||||
|
|
||||||
parser = TextPDFParser(content)
|
parser = TextPDFParser(content)
|
||||||
result = parser.parse()
|
result = parser.parse()
|
||||||
|
```
|
||||||
|
|
||||||
----------------------------------------------------------------------
|
---
|
||||||
Public API Surface
|
|
||||||
----------------------------------------------------------------------
|
# Public API
|
||||||
|
|
||||||
This module re-exports the **recommended public entry points** of OmniRead.
|
This module re-exports the **recommended public entry points** of OmniRead.
|
||||||
|
|
||||||
Consumers are encouraged to import from this namespace rather than from
|
Consumers are encouraged to import from this namespace rather than from
|
||||||
format-specific submodules directly, unless advanced customization is
|
format-specific submodules directly, unless advanced customization is
|
||||||
required.
|
required.
|
||||||
|
|
||||||
Core:
|
- `Content`: Canonical content model.
|
||||||
- Content
|
- `ContentType`: Supported media types.
|
||||||
- ContentType
|
- `HTMLScraper`: HTTP-based HTML acquisition.
|
||||||
|
- `HTMLParser`: Base parser for HTML DOM interpretation.
|
||||||
|
- `FileSystemPDFClient`: Local filesystem PDF access.
|
||||||
|
- `PDFScraper`: PDF-specific content acquisition.
|
||||||
|
- `PDFParser`: Base parser for PDF binary interpretation.
|
||||||
|
- `FileSystemXlsxClient`: Local filesystem spreadsheet access.
|
||||||
|
- `XlsxScraper`: XLSX-specific content acquisition.
|
||||||
|
- `XlsxParser`: Generic string-row parser for xlsx workbooks.
|
||||||
|
|
||||||
HTML:
|
---
|
||||||
- HTMLScraper
|
|
||||||
- HTMLParser
|
|
||||||
|
|
||||||
PDF:
|
# Core Philosophy
|
||||||
- FileSystemPDFClient
|
|
||||||
- PDFScraper
|
`OmniRead` is designed as a **decoupled content engine**:
|
||||||
- PDFParser
|
|
||||||
|
1. **Separation of Concerns**: Scrapers *fetch*, Parsers *interpret*. Neither
|
||||||
|
knows about the other.
|
||||||
|
2. **Normalized Exchange**: All components communicate via the `Content` model,
|
||||||
|
ensuring a consistent contract.
|
||||||
|
3. **Format Agnosticism**: The core logic is independent of whether the input
|
||||||
|
is HTML, PDF, or JSON.
|
||||||
|
|
||||||
|
---
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from .core import Content, ContentType
|
from .core import Content, ContentType
|
||||||
|
from .csv import (
|
||||||
|
BaseCsvClient,
|
||||||
|
CsvParser,
|
||||||
|
CsvParserBase,
|
||||||
|
CsvScraper,
|
||||||
|
FileSystemCsvClient,
|
||||||
|
)
|
||||||
from .html import HTMLScraper, HTMLParser
|
from .html import HTMLScraper, HTMLParser
|
||||||
from .pdf import FileSystemPDFClient, PDFScraper, PDFParser
|
from .pdf import FileSystemPDFClient, PDFScraper, PDFParser
|
||||||
|
from .xlsx import (
|
||||||
|
BaseXlsxClient,
|
||||||
|
FileSystemXlsxClient,
|
||||||
|
XlsxParser,
|
||||||
|
XlsxParserBase,
|
||||||
|
XlsxScraper,
|
||||||
|
)
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
# core
|
# core
|
||||||
"Content",
|
"Content",
|
||||||
"ContentType",
|
"ContentType",
|
||||||
|
|
||||||
# html
|
# html
|
||||||
"HTMLScraper",
|
"HTMLScraper",
|
||||||
"HTMLParser",
|
"HTMLParser",
|
||||||
|
|
||||||
# pdf
|
# pdf
|
||||||
"FileSystemPDFClient",
|
"FileSystemPDFClient",
|
||||||
"PDFScraper",
|
"PDFScraper",
|
||||||
"PDFParser",
|
"PDFParser",
|
||||||
|
# csv
|
||||||
|
"BaseCsvClient",
|
||||||
|
"FileSystemCsvClient",
|
||||||
|
"CsvScraper",
|
||||||
|
"CsvParser",
|
||||||
|
"CsvParserBase",
|
||||||
|
# xlsx
|
||||||
|
"BaseXlsxClient",
|
||||||
|
"FileSystemXlsxClient",
|
||||||
|
"XlsxScraper",
|
||||||
|
"XlsxParser",
|
||||||
|
"XlsxParserBase",
|
||||||
]
|
]
|
||||||
|
|||||||
13
omniread/__init__.pyi
Normal file
13
omniread/__init__.pyi
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
from .core import Content, ContentType
|
||||||
|
from .html import HTMLParser, HTMLScraper
|
||||||
|
from .pdf import FileSystemPDFClient, PDFParser, PDFScraper
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"Content",
|
||||||
|
"ContentType",
|
||||||
|
"HTMLScraper",
|
||||||
|
"HTMLParser",
|
||||||
|
"FileSystemPDFClient",
|
||||||
|
"PDFScraper",
|
||||||
|
"PDFParser",
|
||||||
|
]
|
||||||
@@ -1,4 +1,6 @@
|
|||||||
"""
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
Core domain contracts for OmniRead.
|
Core domain contracts for OmniRead.
|
||||||
|
|
||||||
This package defines the **format-agnostic domain layer** of OmniRead.
|
This package defines the **format-agnostic domain layer** of OmniRead.
|
||||||
@@ -9,11 +11,21 @@ Public exports from this package are considered **stable contracts** and
|
|||||||
are safe for downstream consumers to depend on.
|
are safe for downstream consumers to depend on.
|
||||||
|
|
||||||
Submodules:
|
Submodules:
|
||||||
- content: Canonical content models and enums
|
|
||||||
- parser: Abstract parsing contracts
|
- `content`: Canonical content models and enums.
|
||||||
- scraper: Abstract scraping contracts
|
- `parser`: Abstract parsing contracts.
|
||||||
|
- `scraper`: Abstract scraping contracts.
|
||||||
|
|
||||||
Format-specific behavior must not be introduced at this layer.
|
Format-specific behavior must not be introduced at this layer.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Public API
|
||||||
|
|
||||||
|
- `Content`
|
||||||
|
- `ContentType`
|
||||||
|
|
||||||
|
---
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from .content import Content, ContentType
|
from .content import Content, ContentType
|
||||||
|
|||||||
10
omniread/core/__init__.pyi
Normal file
10
omniread/core/__init__.pyi
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
from .content import Content, ContentType
|
||||||
|
from .parser import BaseParser
|
||||||
|
from .scraper import BaseScraper
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"Content",
|
||||||
|
"ContentType",
|
||||||
|
"BaseParser",
|
||||||
|
"BaseScraper",
|
||||||
|
]
|
||||||
@@ -1,4 +1,6 @@
|
|||||||
"""
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
Canonical content models for OmniRead.
|
Canonical content models for OmniRead.
|
||||||
|
|
||||||
This module defines the **format-agnostic content representation** used across
|
This module defines the **format-agnostic content representation** used across
|
||||||
@@ -9,17 +11,22 @@ retrieved or parsed. Format-specific behavior and metadata must not alter
|
|||||||
the semantic meaning of these models.
|
the semantic meaning of these models.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from enum import Enum
|
from collections.abc import Mapping
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any, Mapping, Optional
|
from enum import Enum
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
class ContentType(str, Enum):
|
class ContentType(str, Enum):
|
||||||
"""
|
"""
|
||||||
Supported MIME types for extracted content.
|
Supported MIME types for extracted content.
|
||||||
|
|
||||||
This enum represents the declared or inferred media type of the content
|
Notes:
|
||||||
source. It is primarily used for routing content to the appropriate
|
**Guarantees:**
|
||||||
|
|
||||||
|
- This enum represents the declared or inferred media type of the
|
||||||
|
content source.
|
||||||
|
- It is primarily used for routing content to the appropriate
|
||||||
parser or downstream consumer.
|
parser or downstream consumer.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@@ -29,6 +36,12 @@ class ContentType(str, Enum):
|
|||||||
PDF = "application/pdf"
|
PDF = "application/pdf"
|
||||||
"""PDF document content."""
|
"""PDF document content."""
|
||||||
|
|
||||||
|
XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||||||
|
"""Office Open XML spreadsheet (xlsx/xlsm) content."""
|
||||||
|
|
||||||
|
CSV = "text/csv"
|
||||||
|
"""Comma-separated-value document content."""
|
||||||
|
|
||||||
JSON = "application/json"
|
JSON = "application/json"
|
||||||
"""JSON document content."""
|
"""JSON document content."""
|
||||||
|
|
||||||
@@ -41,23 +54,31 @@ class Content:
|
|||||||
"""
|
"""
|
||||||
Normalized representation of extracted content.
|
Normalized representation of extracted content.
|
||||||
|
|
||||||
A `Content` instance represents a raw content payload along with minimal
|
Notes:
|
||||||
contextual metadata describing its origin and type.
|
**Responsibilities:**
|
||||||
|
|
||||||
This class is the **primary exchange format** between:
|
- A `Content` instance represents a raw content payload along with
|
||||||
- Scrapers
|
minimal contextual metadata describing its origin and type.
|
||||||
- Parsers
|
- This class is the primary exchange format between scrapers,
|
||||||
- Downstream consumers
|
parsers, and downstream consumers.
|
||||||
|
|
||||||
Attributes:
|
|
||||||
raw: Raw content bytes as retrieved from the source.
|
|
||||||
source: Identifier of the content origin (URL, file path, or logical name).
|
|
||||||
content_type: Optional MIME type of the content, if known.
|
|
||||||
metadata: Optional, implementation-defined metadata associated with
|
|
||||||
the content (e.g., headers, encoding hints, extraction notes).
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
raw: bytes
|
raw: bytes
|
||||||
|
"""
|
||||||
|
Raw content bytes as retrieved from the source.
|
||||||
|
"""
|
||||||
|
|
||||||
source: str
|
source: str
|
||||||
content_type: Optional[ContentType] = None
|
"""
|
||||||
metadata: Optional[Mapping[str, Any]] = None
|
Identifier of the content origin (URL, file path, or logical name).
|
||||||
|
"""
|
||||||
|
|
||||||
|
content_type: ContentType | None = None
|
||||||
|
"""
|
||||||
|
Optional MIME type of the content, if known.
|
||||||
|
"""
|
||||||
|
|
||||||
|
metadata: Mapping[str, Any] | None = None
|
||||||
|
"""
|
||||||
|
Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes).
|
||||||
|
"""
|
||||||
|
|||||||
24
omniread/core/content.pyi
Normal file
24
omniread/core/content.pyi
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
from collections.abc import Mapping
|
||||||
|
from enum import Enum
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
class ContentType(str, Enum):
|
||||||
|
HTML = "text/html"
|
||||||
|
PDF = "application/pdf"
|
||||||
|
XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||||||
|
CSV = "text/csv"
|
||||||
|
JSON = "application/json"
|
||||||
|
XML = "application/xml"
|
||||||
|
|
||||||
|
class Content:
|
||||||
|
raw: bytes
|
||||||
|
source: str
|
||||||
|
content_type: ContentType | None
|
||||||
|
metadata: Mapping[str, Any] | None
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
raw: bytes,
|
||||||
|
source: str,
|
||||||
|
content_type: ContentType | None = ...,
|
||||||
|
metadata: Mapping[str, Any] | None = ...,
|
||||||
|
) -> None: ...
|
||||||
@@ -1,22 +1,26 @@
|
|||||||
"""
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
Abstract parsing contracts for OmniRead.
|
Abstract parsing contracts for OmniRead.
|
||||||
|
|
||||||
This module defines the **format-agnostic parser interface** used to transform
|
This module defines the **format-agnostic parser interface** used to transform
|
||||||
raw content into structured, typed representations.
|
raw content into structured, typed representations.
|
||||||
|
|
||||||
Parsers are responsible for:
|
Parsers are responsible for:
|
||||||
|
|
||||||
- Interpreting a single `Content` instance
|
- Interpreting a single `Content` instance
|
||||||
- Validating compatibility with the content type
|
- Validating compatibility with the content type
|
||||||
- Producing a structured output suitable for downstream consumers
|
- Producing a structured output suitable for downstream consumers
|
||||||
|
|
||||||
Parsers are not responsible for:
|
Parsers are not responsible for:
|
||||||
|
|
||||||
- Fetching or acquiring content
|
- Fetching or acquiring content
|
||||||
- Performing retries or error recovery
|
- Performing retries or error recovery
|
||||||
- Managing multiple content sources
|
- Managing multiple content sources
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from typing import Generic, TypeVar, Set
|
from typing import Generic, TypeVar
|
||||||
|
|
||||||
from .content import Content, ContentType
|
from .content import Content, ContentType
|
||||||
|
|
||||||
@@ -27,23 +31,24 @@ class BaseParser(ABC, Generic[T]):
|
|||||||
"""
|
"""
|
||||||
Base interface for all parsers.
|
Base interface for all parsers.
|
||||||
|
|
||||||
A parser is a self-contained object that owns the Content
|
Notes:
|
||||||
it is responsible for interpreting.
|
**Guarantees:**
|
||||||
|
|
||||||
Implementations must:
|
- A parser is a self-contained object that owns the `Content` it is
|
||||||
- Declare supported content types via `supported_types`
|
responsible for interpreting.
|
||||||
- Raise parsing-specific exceptions from `parse()`
|
- Consumers may rely on early validation of content compatibility
|
||||||
- Remain deterministic for a given input
|
and type-stable return values from `parse()`.
|
||||||
|
|
||||||
Consumers may rely on:
|
**Responsibilities:**
|
||||||
- Early validation of content compatibility
|
|
||||||
- Type-stable return values from `parse()`
|
- Implementations must declare supported content types via `supported_types`.
|
||||||
|
- Implementations must raise parsing-specific exceptions from `parse()`.
|
||||||
|
- Implementations must remain deterministic for a given input.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
supported_types: Set[ContentType] = set()
|
supported_types: set[ContentType] = set()
|
||||||
"""Set of content types supported by this parser.
|
"""
|
||||||
|
Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic.
|
||||||
An empty set indicates that the parser is content-type agnostic.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, content: Content):
|
def __init__(self, content: Content):
|
||||||
@@ -51,10 +56,12 @@ class BaseParser(ABC, Generic[T]):
|
|||||||
Initialize the parser with content to be parsed.
|
Initialize the parser with content to be parsed.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
content: Content instance to be parsed.
|
content (Content):
|
||||||
|
Content instance to be parsed.
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
ValueError: If the content type is not supported by this parser.
|
ValueError:
|
||||||
|
If the content type is not supported by this parser.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
self.content = content
|
self.content = content
|
||||||
@@ -70,14 +77,19 @@ class BaseParser(ABC, Generic[T]):
|
|||||||
"""
|
"""
|
||||||
Parse the owned content into structured output.
|
Parse the owned content into structured output.
|
||||||
|
|
||||||
Implementations must fully consume the provided content and
|
|
||||||
return a deterministic, structured output.
|
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
|
T:
|
||||||
Parsed, structured representation.
|
Parsed, structured representation.
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
Exception: Parsing-specific errors as defined by the implementation.
|
Exception:
|
||||||
|
Parsing-specific errors as defined by the implementation.
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
**Responsibilities:**
|
||||||
|
|
||||||
|
- Implementations must fully consume the provided content and
|
||||||
|
return a deterministic, structured output.
|
||||||
"""
|
"""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
@@ -86,6 +98,7 @@ class BaseParser(ABC, Generic[T]):
|
|||||||
Check whether this parser supports the content's type.
|
Check whether this parser supports the content's type.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
|
bool:
|
||||||
True if the content type is supported; False otherwise.
|
True if the content type is supported; False otherwise.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|||||||
14
omniread/core/parser.pyi
Normal file
14
omniread/core/parser.pyi
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
from abc import ABC, abstractmethod
|
||||||
|
from typing import Generic, TypeVar
|
||||||
|
|
||||||
|
from .content import Content, ContentType
|
||||||
|
|
||||||
|
T = TypeVar("T")
|
||||||
|
|
||||||
|
class BaseParser(ABC, Generic[T]):
|
||||||
|
supported_types: set[ContentType]
|
||||||
|
content: Content
|
||||||
|
def __init__(self, content: Content) -> None: ...
|
||||||
|
@abstractmethod
|
||||||
|
def parse(self) -> T: ...
|
||||||
|
def supports(self) -> bool: ...
|
||||||
@@ -1,15 +1,19 @@
|
|||||||
"""
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
Abstract scraping contracts for OmniRead.
|
Abstract scraping contracts for OmniRead.
|
||||||
|
|
||||||
This module defines the **format-agnostic scraper interface** responsible for
|
This module defines the **format-agnostic scraper interface** responsible for
|
||||||
acquiring raw content from external sources.
|
acquiring raw content from external sources.
|
||||||
|
|
||||||
Scrapers are responsible for:
|
Scrapers are responsible for:
|
||||||
|
|
||||||
- Locating and retrieving raw content bytes
|
- Locating and retrieving raw content bytes
|
||||||
- Attaching minimal contextual metadata
|
- Attaching minimal contextual metadata
|
||||||
- Returning normalized `Content` objects
|
- Returning normalized `Content` objects
|
||||||
|
|
||||||
Scrapers are explicitly NOT responsible for:
|
Scrapers are explicitly NOT responsible for:
|
||||||
|
|
||||||
- Parsing or interpreting content
|
- Parsing or interpreting content
|
||||||
- Inferring structure or semantics
|
- Inferring structure or semantics
|
||||||
- Performing content-type specific processing
|
- Performing content-type specific processing
|
||||||
@@ -18,7 +22,8 @@ All interpretation must be delegated to parsers.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from typing import Any, Mapping, Optional
|
from collections.abc import Mapping
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
from .content import Content
|
from .content import Content
|
||||||
|
|
||||||
@@ -27,23 +32,21 @@ class BaseScraper(ABC):
|
|||||||
"""
|
"""
|
||||||
Base interface for all scrapers.
|
Base interface for all scrapers.
|
||||||
|
|
||||||
A scraper is responsible ONLY for fetching raw content
|
Notes:
|
||||||
(bytes) from a source. It must not interpret or parse it.
|
**Responsibilities:**
|
||||||
|
|
||||||
A scraper is a **stateless acquisition component** that retrieves raw
|
- A scraper is responsible ONLY for fetching raw content (bytes)
|
||||||
|
from a source. It must not interpret or parse it.
|
||||||
|
- A scraper is a stateless acquisition component that retrieves raw
|
||||||
content from a source and returns it as a `Content` object.
|
content from a source and returns it as a `Content` object.
|
||||||
|
- Scrapers define how content is obtained, not what the content means.
|
||||||
|
- Implementations may vary in transport mechanism, authentication
|
||||||
|
strategy, retry and backoff behavior.
|
||||||
|
|
||||||
Scrapers define *how content is obtained*, not *what the content means*.
|
**Constraints:**
|
||||||
|
|
||||||
Implementations may vary in:
|
- Implementations must not parse content, modify content semantics,
|
||||||
- Transport mechanism (HTTP, filesystem, cloud storage)
|
or couple scraping logic to a specific parser.
|
||||||
- Authentication strategy
|
|
||||||
- Retry and backoff behavior
|
|
||||||
|
|
||||||
Implementations must not:
|
|
||||||
- Parse content
|
|
||||||
- Modify content semantics
|
|
||||||
- Couple scraping logic to a specific parser
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
@@ -51,25 +54,30 @@ class BaseScraper(ABC):
|
|||||||
self,
|
self,
|
||||||
source: str,
|
source: str,
|
||||||
*,
|
*,
|
||||||
metadata: Optional[Mapping[str, Any]] = None,
|
metadata: Mapping[str, Any] | None = None,
|
||||||
) -> Content:
|
) -> Content:
|
||||||
"""
|
"""
|
||||||
Fetch raw content from the given source.
|
Fetch raw content from the given source.
|
||||||
|
|
||||||
Implementations must retrieve the content referenced by `source`
|
|
||||||
and return it as raw bytes wrapped in a `Content` object.
|
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
source: Location identifier (URL, file path, S3 URI, etc.)
|
source (str):
|
||||||
metadata: Optional hints for the scraper (headers, auth, etc.)
|
Location identifier (URL, file path, S3 URI, etc.).
|
||||||
|
|
||||||
|
metadata (Mapping[str, Any] | None, optional):
|
||||||
|
Optional hints for the scraper (headers, auth, etc.).
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
|
Content:
|
||||||
Content object containing raw bytes and metadata.
|
Content object containing raw bytes and metadata.
|
||||||
- Raw content bytes
|
|
||||||
- Source identifier
|
|
||||||
- Optional metadata
|
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
Exception: Retrieval-specific errors as defined by the implementation.
|
Exception:
|
||||||
|
Retrieval-specific errors as defined by the implementation.
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
**Responsibilities:**
|
||||||
|
|
||||||
|
- Implementations must retrieve the content referenced by `source`
|
||||||
|
and return it as raw bytes wrapped in a `Content` object.
|
||||||
"""
|
"""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|||||||
11
omniread/core/scraper.pyi
Normal file
11
omniread/core/scraper.pyi
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
from abc import ABC, abstractmethod
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from .content import Content
|
||||||
|
|
||||||
|
class BaseScraper(ABC):
|
||||||
|
@abstractmethod
|
||||||
|
def fetch(
|
||||||
|
self, source: str, *, metadata: Mapping[str, Any] | None = ...
|
||||||
|
) -> Content: ...
|
||||||
26
omniread/csv/__init__.py
Normal file
26
omniread/csv/__init__.py
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
|
CSV subpackage for OmniRead.
|
||||||
|
|
||||||
|
Provides acquisition and parsing of comma-separated-value content:
|
||||||
|
|
||||||
|
- `BaseCsvClient`: abstract backing-store client for csv bytes.
|
||||||
|
- `FileSystemCsvClient`: local filesystem implementation.
|
||||||
|
- `CsvScraper`: wraps fetched bytes into canonical `Content`.
|
||||||
|
- `CsvParserBase`: content-type-enforcing parser contract.
|
||||||
|
- `CsvParser`: generic string-row parser built on the standard csv module.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from .client import BaseCsvClient, FileSystemCsvClient
|
||||||
|
from .parser import CsvParser
|
||||||
|
from .parser_base import CsvParserBase
|
||||||
|
from .scraper import CsvScraper
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"BaseCsvClient",
|
||||||
|
"FileSystemCsvClient",
|
||||||
|
"CsvScraper",
|
||||||
|
"CsvParser",
|
||||||
|
"CsvParserBase",
|
||||||
|
]
|
||||||
12
omniread/csv/__init__.pyi
Normal file
12
omniread/csv/__init__.pyi
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
from .client import BaseCsvClient, FileSystemCsvClient
|
||||||
|
from .parser import CsvParser
|
||||||
|
from .parser_base import CsvParserBase
|
||||||
|
from .scraper import CsvScraper
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"BaseCsvClient",
|
||||||
|
"FileSystemCsvClient",
|
||||||
|
"CsvScraper",
|
||||||
|
"CsvParser",
|
||||||
|
"CsvParserBase",
|
||||||
|
]
|
||||||
97
omniread/csv/client.py
Normal file
97
omniread/csv/client.py
Normal file
@@ -0,0 +1,97 @@
|
|||||||
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
|
CSV client abstractions for OmniRead.
|
||||||
|
|
||||||
|
This module defines the **client layer** responsible for retrieving raw
|
||||||
|
comma-separated-value document bytes from a concrete backing store.
|
||||||
|
|
||||||
|
Clients provide low-level access to csv binaries and are intentionally
|
||||||
|
decoupled from scraping and parsing logic. They do not perform validation,
|
||||||
|
interpretation, or content extraction.
|
||||||
|
|
||||||
|
Typical backing stores include:
|
||||||
|
|
||||||
|
- Local filesystems
|
||||||
|
- Object storage (S3, GCS, etc.)
|
||||||
|
- Network file systems
|
||||||
|
"""
|
||||||
|
|
||||||
|
from abc import ABC, abstractmethod
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
class BaseCsvClient(ABC):
|
||||||
|
"""
|
||||||
|
Abstract client responsible for retrieving csv bytes.
|
||||||
|
|
||||||
|
Retrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
**Responsibilities:**
|
||||||
|
|
||||||
|
- Implementations must accept a source identifier appropriate to
|
||||||
|
the backing store.
|
||||||
|
- Return the full csv binary payload.
|
||||||
|
- Raise retrieval-specific errors on failure.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def fetch(self, source: Any) -> bytes:
|
||||||
|
"""
|
||||||
|
Fetch raw csv bytes from the given source.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
source (Any):
|
||||||
|
Identifier of the csv location, such as a file path,
|
||||||
|
object storage key, or remote reference.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
bytes:
|
||||||
|
Raw csv bytes.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
Exception:
|
||||||
|
Retrieval-specific errors defined by the implementation.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
|
||||||
|
class FileSystemCsvClient(BaseCsvClient):
|
||||||
|
"""
|
||||||
|
CSV client that reads from the local filesystem.
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
**Guarantees:**
|
||||||
|
|
||||||
|
- This client reads csv files directly from the disk and
|
||||||
|
returns their raw binary contents.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def fetch(self, path: Path) -> bytes:
|
||||||
|
"""
|
||||||
|
Read a csv file from the local filesystem.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
path (Path):
|
||||||
|
Filesystem path to the csv file.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
bytes:
|
||||||
|
Raw csv bytes.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
FileNotFoundError:
|
||||||
|
If the path does not exist.
|
||||||
|
ValueError:
|
||||||
|
If the path exists but is not a file.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if not path.exists():
|
||||||
|
raise FileNotFoundError(f"csv not found: {path}")
|
||||||
|
|
||||||
|
if not path.is_file():
|
||||||
|
raise ValueError(f"Path is not a file: {path}")
|
||||||
|
|
||||||
|
return path.read_bytes()
|
||||||
10
omniread/csv/client.pyi
Normal file
10
omniread/csv/client.pyi
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
from abc import ABC, abstractmethod
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
class BaseCsvClient(ABC):
|
||||||
|
@abstractmethod
|
||||||
|
def fetch(self, source: Any) -> bytes: ...
|
||||||
|
|
||||||
|
class FileSystemCsvClient(BaseCsvClient):
|
||||||
|
def fetch(self, path: Path) -> bytes: ...
|
||||||
102
omniread/csv/parser.py
Normal file
102
omniread/csv/parser.py
Normal file
@@ -0,0 +1,102 @@
|
|||||||
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
|
CSV parser implementations for OmniRead.
|
||||||
|
|
||||||
|
This module provides a concrete, generic parser for comma-separated-value
|
||||||
|
documents. It exposes records as lists of string cells so downstream
|
||||||
|
consumers can interpret tabular content without depending on the ``csv``
|
||||||
|
module directly.
|
||||||
|
|
||||||
|
The parser is intentionally statement-agnostic: it performs no header
|
||||||
|
detection or column interpretation beyond basic cell normalization and
|
||||||
|
delimiter detection.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from csv import Sniffer, reader
|
||||||
|
from io import StringIO
|
||||||
|
|
||||||
|
from omniread.core.content import Content
|
||||||
|
|
||||||
|
from .parser_base import CsvParserBase
|
||||||
|
|
||||||
|
|
||||||
|
class CsvParser(CsvParserBase[list[list[str]]]):
|
||||||
|
"""
|
||||||
|
Generic csv parser producing string rows from the document.
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
**Responsibilities:**
|
||||||
|
|
||||||
|
- Decode the payload (UTF-8 with BOM support, Latin-1 fallback).
|
||||||
|
- Detect the delimiter from a leading sample (`,` `;` tab `|`),
|
||||||
|
defaulting to `,`.
|
||||||
|
- Normalize cells into deterministic stripped string values.
|
||||||
|
- Expose row extraction helpers mirroring `XlsxParser.rows`.
|
||||||
|
|
||||||
|
**Constraints:**
|
||||||
|
|
||||||
|
- All values are strings; consumers requiring typed values must
|
||||||
|
convert on their side.
|
||||||
|
- Quoted fields containing delimiters/newlines are handled by
|
||||||
|
the standard ``csv`` module.
|
||||||
|
"""
|
||||||
|
|
||||||
|
_DELIMITERS = ",;\t|"
|
||||||
|
|
||||||
|
def __init__(self, content: Content):
|
||||||
|
"""
|
||||||
|
Initialize the parser.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
content (Content):
|
||||||
|
CSV content to parse; its type must be supported.
|
||||||
|
"""
|
||||||
|
super().__init__(content)
|
||||||
|
|
||||||
|
def parse(self) -> list[list[str]]:
|
||||||
|
"""
|
||||||
|
Parse the document into normalized string rows.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
list[list[str]]:
|
||||||
|
Rows of the document.
|
||||||
|
"""
|
||||||
|
return self.rows()
|
||||||
|
|
||||||
|
def rows(self, *, skip_empty: bool = True) -> list[list[str]]:
|
||||||
|
"""
|
||||||
|
Extract normalized string rows from the document.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
skip_empty (bool):
|
||||||
|
When True (default), rows whose cells are all blank are
|
||||||
|
omitted.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
list[list[str]]:
|
||||||
|
Normalized rows; trailing blank cells are trimmed per row.
|
||||||
|
"""
|
||||||
|
text = self._decode(self.content.raw)
|
||||||
|
dialect_sample = text[:4096]
|
||||||
|
try:
|
||||||
|
dialect = Sniffer().sniff(dialect_sample, delimiters=self._DELIMITERS)
|
||||||
|
delimiter = dialect.delimiter
|
||||||
|
except Exception:
|
||||||
|
delimiter = ","
|
||||||
|
out: list[list[str]] = []
|
||||||
|
for row in reader(StringIO(text), delimiter=delimiter):
|
||||||
|
cells = [c.strip() for c in row]
|
||||||
|
while cells and not cells[-1]:
|
||||||
|
cells.pop()
|
||||||
|
if skip_empty and not any(cells):
|
||||||
|
continue
|
||||||
|
out.append(cells)
|
||||||
|
return out
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _decode(raw: bytes) -> str:
|
||||||
|
try:
|
||||||
|
return raw.decode("utf-8-sig")
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
return raw.decode("latin-1")
|
||||||
8
omniread/csv/parser.pyi
Normal file
8
omniread/csv/parser.pyi
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
from omniread.core.content import Content
|
||||||
|
|
||||||
|
from .parser_base import CsvParserBase
|
||||||
|
|
||||||
|
class CsvParser(CsvParserBase[list[list[str]]]):
|
||||||
|
def __init__(self, content: Content) -> None: ...
|
||||||
|
def parse(self) -> list[list[str]]: ...
|
||||||
|
def rows(self, *, skip_empty: bool = True) -> list[list[str]]: ...
|
||||||
55
omniread/csv/parser_base.py
Normal file
55
omniread/csv/parser_base.py
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
|
CSV parser base implementation for OmniRead.
|
||||||
|
|
||||||
|
This module defines the **CSV-specific parser contract**, extending the
|
||||||
|
format-agnostic `BaseParser` with constraints appropriate for
|
||||||
|
comma-separated-value documents.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from abc import abstractmethod
|
||||||
|
from typing import Generic, TypeVar
|
||||||
|
|
||||||
|
from omniread.core.content import ContentType
|
||||||
|
from omniread.core.parser import BaseParser
|
||||||
|
|
||||||
|
T = TypeVar("T")
|
||||||
|
|
||||||
|
|
||||||
|
class CsvParserBase(BaseParser[T], Generic[T]):
|
||||||
|
"""
|
||||||
|
Base csv parser.
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
**Responsibilities:**
|
||||||
|
|
||||||
|
- This class enforces csv content-type compatibility and provides
|
||||||
|
the extension point for implementing concrete csv parsing
|
||||||
|
strategies.
|
||||||
|
|
||||||
|
**Constraints:**
|
||||||
|
|
||||||
|
- Concrete implementations must define the output type `T` and
|
||||||
|
implement the `parse()` method.
|
||||||
|
"""
|
||||||
|
|
||||||
|
supported_types = {ContentType.CSV}
|
||||||
|
"""
|
||||||
|
Set of content types supported by this parser (CSV only).
|
||||||
|
"""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def parse(self) -> T:
|
||||||
|
"""
|
||||||
|
Parse csv content into a structured output.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
T:
|
||||||
|
Parsed representation of type `T`.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
Exception:
|
||||||
|
Parsing-specific errors as defined by the implementation.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
13
omniread/csv/parser_base.pyi
Normal file
13
omniread/csv/parser_base.pyi
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
from abc import abstractmethod
|
||||||
|
from typing import Generic, TypeVar
|
||||||
|
|
||||||
|
from omniread.core.content import ContentType
|
||||||
|
from omniread.core.parser import BaseParser
|
||||||
|
|
||||||
|
T = TypeVar("T")
|
||||||
|
|
||||||
|
class CsvParserBase(BaseParser[T], Generic[T]):
|
||||||
|
supported_types: set[ContentType]
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def parse(self) -> T: ...
|
||||||
81
omniread/csv/scraper.py
Normal file
81
omniread/csv/scraper.py
Normal file
@@ -0,0 +1,81 @@
|
|||||||
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
|
CSV scraper for OmniRead.
|
||||||
|
|
||||||
|
This module defines the scraper responsible for acquiring raw
|
||||||
|
comma-separated-value document content from a backing store via a
|
||||||
|
configured client.
|
||||||
|
|
||||||
|
The scraper does not interpret or parse the acquired bytes; it wraps them in
|
||||||
|
the canonical `Content` model.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from omniread.core.content import Content, ContentType
|
||||||
|
|
||||||
|
from .client import BaseCsvClient
|
||||||
|
|
||||||
|
|
||||||
|
class CsvScraper:
|
||||||
|
"""
|
||||||
|
Scraper for csv documents.
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
**Responsibilities:**
|
||||||
|
|
||||||
|
- Fetch raw csv bytes via the configured client.
|
||||||
|
- Wrap the payload in a canonical `Content` instance with the
|
||||||
|
CSV content type and source identifier.
|
||||||
|
|
||||||
|
**Constraints:**
|
||||||
|
|
||||||
|
- The scraper does not perform parsing or interpretation.
|
||||||
|
- Does not assume a specific storage backend.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, *, client: BaseCsvClient):
|
||||||
|
"""
|
||||||
|
Initialize the CSV scraper.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
client (BaseCsvClient):
|
||||||
|
Client responsible for retrieving raw csv bytes.
|
||||||
|
"""
|
||||||
|
self._client = client
|
||||||
|
|
||||||
|
def fetch(
|
||||||
|
self,
|
||||||
|
source: Any,
|
||||||
|
*,
|
||||||
|
metadata: Mapping[str, Any] | None = None,
|
||||||
|
) -> Content:
|
||||||
|
"""
|
||||||
|
Fetch a csv document from the given source.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
source (Any):
|
||||||
|
Identifier of the csv source as understood by the
|
||||||
|
configured client.
|
||||||
|
metadata (Mapping[str, Any] | None, optional):
|
||||||
|
Optional metadata to attach to the returned content.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Content:
|
||||||
|
A `Content` instance containing raw csv bytes, source
|
||||||
|
identifier, CSV content type, and optional metadata.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
Exception:
|
||||||
|
Retrieval-specific errors raised by the client.
|
||||||
|
"""
|
||||||
|
raw = self._client.fetch(source)
|
||||||
|
|
||||||
|
return Content(
|
||||||
|
raw=raw,
|
||||||
|
source=source,
|
||||||
|
content_type=ContentType.CSV,
|
||||||
|
metadata=dict(metadata) if metadata else None,
|
||||||
|
)
|
||||||
12
omniread/csv/scraper.pyi
Normal file
12
omniread/csv/scraper.pyi
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
from collections.abc import Mapping
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from omniread.core.content import Content
|
||||||
|
|
||||||
|
from .client import BaseCsvClient
|
||||||
|
|
||||||
|
class CsvScraper:
|
||||||
|
def __init__(self, *, client: BaseCsvClient) -> None: ...
|
||||||
|
def fetch(
|
||||||
|
self, source: Any, *, metadata: Mapping[str, Any] | None = ...
|
||||||
|
) -> Content: ...
|
||||||
@@ -1,22 +1,34 @@
|
|||||||
"""
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
HTML format implementation for OmniRead.
|
HTML format implementation for OmniRead.
|
||||||
|
|
||||||
This package provides **HTML-specific implementations** of the core OmniRead
|
This package provides **HTML-specific implementations** of the core OmniRead
|
||||||
contracts defined in `omniread.core`.
|
contracts defined in `omniread.core`.
|
||||||
|
|
||||||
It includes:
|
It includes:
|
||||||
- HTML parsers that interpret HTML content
|
|
||||||
- HTML scrapers that retrieve HTML documents
|
|
||||||
|
|
||||||
This package:
|
- HTML parsers that interpret HTML content.
|
||||||
- Implements, but does not redefine, core contracts
|
- HTML scrapers that retrieve HTML documents.
|
||||||
- May contain HTML-specific behavior and edge-case handling
|
|
||||||
- Produces canonical content models defined in `omniread.core.content`
|
Key characteristics:
|
||||||
|
|
||||||
|
- Implements, but does not redefine, core contracts.
|
||||||
|
- May contain HTML-specific behavior and edge-case handling.
|
||||||
|
- Produces canonical content models defined in `omniread.core.content`.
|
||||||
|
|
||||||
Consumers should depend on `omniread.core` interfaces wherever possible and
|
Consumers should depend on `omniread.core` interfaces wherever possible and
|
||||||
use this package only when HTML-specific behavior is required.
|
use this package only when HTML-specific behavior is required.
|
||||||
"""
|
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Public API
|
||||||
|
|
||||||
|
- `HTMLScraper`
|
||||||
|
- `HTMLParser`
|
||||||
|
|
||||||
|
---
|
||||||
|
"""
|
||||||
|
|
||||||
from .scraper import HTMLScraper
|
from .scraper import HTMLScraper
|
||||||
from .parser import HTMLParser
|
from .parser import HTMLParser
|
||||||
|
|||||||
4
omniread/html/__init__.pyi
Normal file
4
omniread/html/__init__.pyi
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
from .parser import HTMLParser
|
||||||
|
from .scraper import HTMLScraper
|
||||||
|
|
||||||
|
__all__ = ["HTMLScraper", "HTMLParser"]
|
||||||
@@ -1,10 +1,13 @@
|
|||||||
"""
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
HTML parser base implementations for OmniRead.
|
HTML parser base implementations for OmniRead.
|
||||||
|
|
||||||
This module provides reusable HTML parsing utilities built on top of
|
This module provides reusable HTML parsing utilities built on top of
|
||||||
the abstract parser contracts defined in `omniread.core.parser`.
|
the abstract parser contracts defined in `omniread.core.parser`.
|
||||||
|
|
||||||
It supplies:
|
It supplies:
|
||||||
|
|
||||||
- Content-type enforcement for HTML inputs
|
- Content-type enforcement for HTML inputs
|
||||||
- BeautifulSoup initialization and lifecycle management
|
- BeautifulSoup initialization and lifecycle management
|
||||||
- Common helper methods for extracting structured data from HTML elements
|
- Common helper methods for extracting structured data from HTML elements
|
||||||
@@ -13,12 +16,12 @@ Concrete parsers must subclass `HTMLParser` and implement the `parse()` method
|
|||||||
to return a structured representation appropriate for their use case.
|
to return a structured representation appropriate for their use case.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Any, Generic, TypeVar, Optional
|
|
||||||
from abc import abstractmethod
|
from abc import abstractmethod
|
||||||
|
from typing import Any, Generic, TypeVar
|
||||||
|
|
||||||
from bs4 import BeautifulSoup, Tag
|
from bs4 import BeautifulSoup, Tag
|
||||||
|
|
||||||
from omniread.core.content import ContentType, Content
|
from omniread.core.content import Content, ContentType
|
||||||
from omniread.core.parser import BaseParser
|
from omniread.core.parser import BaseParser
|
||||||
|
|
||||||
T = TypeVar("T")
|
T = TypeVar("T")
|
||||||
@@ -28,36 +31,44 @@ class HTMLParser(BaseParser[T], Generic[T]):
|
|||||||
"""
|
"""
|
||||||
Base HTML parser.
|
Base HTML parser.
|
||||||
|
|
||||||
This class extends the core `BaseParser` with HTML-specific behavior,
|
Notes:
|
||||||
|
**Responsibilities:**
|
||||||
|
|
||||||
|
- This class extends the core `BaseParser` with HTML-specific behavior,
|
||||||
including DOM parsing via BeautifulSoup and reusable extraction helpers.
|
including DOM parsing via BeautifulSoup and reusable extraction helpers.
|
||||||
|
- Provides reusable helpers for HTML extraction. Concrete parsers must
|
||||||
|
explicitly define the return type.
|
||||||
|
|
||||||
Provides reusable helpers for HTML extraction.
|
**Guarantees:**
|
||||||
Concrete parsers must explicitly define the return type.
|
|
||||||
|
|
||||||
Characteristics:
|
- Accepts only HTML content.
|
||||||
- Accepts only HTML content
|
- Owns a parsed BeautifulSoup DOM tree.
|
||||||
- Owns a parsed BeautifulSoup DOM tree
|
- Provides pure helper utilities for common HTML structures.
|
||||||
- Provides pure helper utilities for common HTML structures
|
|
||||||
|
|
||||||
Concrete subclasses must:
|
**Constraints:**
|
||||||
- Define the output type `T`
|
|
||||||
- Implement the `parse()` method
|
- Concrete subclasses must define the output type `T` and implement
|
||||||
|
the `parse()` method.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
supported_types = {ContentType.HTML}
|
supported_types = {ContentType.HTML}
|
||||||
"""Set of content types supported by this parser (HTML only)."""
|
"""
|
||||||
|
Set of content types supported by this parser (HTML only).
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, content: Content, features: str = "html.parser"):
|
def __init__(self, content: Content, features: str = "html.parser"):
|
||||||
"""
|
"""
|
||||||
Initialize the HTML parser.
|
Initialize the HTML parser.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
content: HTML content to be parsed.
|
content (Content):
|
||||||
features: BeautifulSoup parser backend to use
|
HTML content to be parsed.
|
||||||
(e.g., 'html.parser', 'lxml').
|
features (str, optional):
|
||||||
|
BeautifulSoup parser backend to use (e.g., 'html.parser', 'lxml').
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
ValueError: If the content is empty or not valid HTML.
|
ValueError:
|
||||||
|
If the content is empty or not valid HTML.
|
||||||
"""
|
"""
|
||||||
super().__init__(content)
|
super().__init__(content)
|
||||||
self._features = features
|
self._features = features
|
||||||
@@ -72,11 +83,15 @@ class HTMLParser(BaseParser[T], Generic[T]):
|
|||||||
"""
|
"""
|
||||||
Fully parse the HTML content into structured output.
|
Fully parse the HTML content into structured output.
|
||||||
|
|
||||||
Implementations must fully interpret the HTML DOM and return
|
|
||||||
a deterministic, structured output.
|
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
|
T:
|
||||||
Parsed representation of type `T`.
|
Parsed representation of type `T`.
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
**Responsibilities:**
|
||||||
|
|
||||||
|
- Implementations must fully interpret the HTML DOM and return a
|
||||||
|
deterministic, structured output.
|
||||||
"""
|
"""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
@@ -90,23 +105,28 @@ class HTMLParser(BaseParser[T], Generic[T]):
|
|||||||
Extract normalized text from a `<div>` element.
|
Extract normalized text from a `<div>` element.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
div: BeautifulSoup tag representing a `<div>`.
|
div (Tag):
|
||||||
separator: String used to separate text nodes.
|
BeautifulSoup tag representing a `<div>`.
|
||||||
|
separator (str, optional):
|
||||||
|
String used to separate text nodes.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
|
str:
|
||||||
Flattened, whitespace-normalized text content.
|
Flattened, whitespace-normalized text content.
|
||||||
"""
|
"""
|
||||||
return div.get_text(separator=separator, strip=True)
|
return div.get_text(separator=separator, strip=True)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def parse_link(a: Tag) -> Optional[str]:
|
def parse_link(a: Tag) -> str | None:
|
||||||
"""
|
"""
|
||||||
Extract the hyperlink reference from an `<a>` element.
|
Extract the hyperlink reference from an `<a>` element.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
a: BeautifulSoup tag representing an anchor.
|
a (Tag):
|
||||||
|
BeautifulSoup tag representing an anchor.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
|
str | None:
|
||||||
The value of the `href` attribute, or None if absent.
|
The value of the `href` attribute, or None if absent.
|
||||||
"""
|
"""
|
||||||
return a.get("href")
|
return a.get("href")
|
||||||
@@ -117,17 +137,16 @@ class HTMLParser(BaseParser[T], Generic[T]):
|
|||||||
Parse an HTML table into a 2D list of strings.
|
Parse an HTML table into a 2D list of strings.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
table: BeautifulSoup tag representing a `<table>`.
|
table (Tag):
|
||||||
|
BeautifulSoup tag representing a `<table>`.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
|
list[list[str]]:
|
||||||
A list of rows, where each row is a list of cell text values.
|
A list of rows, where each row is a list of cell text values.
|
||||||
"""
|
"""
|
||||||
rows: list[list[str]] = []
|
rows: list[list[str]] = []
|
||||||
for tr in table.find_all("tr"):
|
for tr in table.find_all("tr"):
|
||||||
cells = [
|
cells = [cell.get_text(strip=True) for cell in tr.find_all(["td", "th"])]
|
||||||
cell.get_text(strip=True)
|
|
||||||
for cell in tr.find_all(["td", "th"])
|
|
||||||
]
|
|
||||||
if cells:
|
if cells:
|
||||||
rows.append(cells)
|
rows.append(cells)
|
||||||
return rows
|
return rows
|
||||||
@@ -141,10 +160,12 @@ class HTMLParser(BaseParser[T], Generic[T]):
|
|||||||
Build a BeautifulSoup DOM tree from raw HTML content.
|
Build a BeautifulSoup DOM tree from raw HTML content.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
|
BeautifulSoup:
|
||||||
Parsed BeautifulSoup document tree.
|
Parsed BeautifulSoup document tree.
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
ValueError: If the content payload is empty.
|
ValueError:
|
||||||
|
If the content payload is empty.
|
||||||
"""
|
"""
|
||||||
if not self.content.raw:
|
if not self.content.raw:
|
||||||
raise ValueError("Empty HTML content")
|
raise ValueError("Empty HTML content")
|
||||||
@@ -154,12 +175,16 @@ class HTMLParser(BaseParser[T], Generic[T]):
|
|||||||
"""
|
"""
|
||||||
Extract high-level metadata from the HTML document.
|
Extract high-level metadata from the HTML document.
|
||||||
|
|
||||||
This includes:
|
|
||||||
- Document title
|
|
||||||
- `<meta>` tag name/property → content mappings
|
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
|
dict[str, Any]:
|
||||||
Dictionary containing extracted metadata.
|
Dictionary containing extracted metadata.
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
**Responsibilities:**
|
||||||
|
|
||||||
|
- Extract high-level metadata from the HTML document.
|
||||||
|
- This includes: Document title, `<meta>` tag name/property to
|
||||||
|
content mappings.
|
||||||
"""
|
"""
|
||||||
soup = self._soup
|
soup = self._soup
|
||||||
|
|
||||||
|
|||||||
20
omniread/html/parser.pyi
Normal file
20
omniread/html/parser.pyi
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
from typing import Any, Generic, TypeVar
|
||||||
|
|
||||||
|
from bs4 import Tag
|
||||||
|
|
||||||
|
from omniread.core.content import Content, ContentType
|
||||||
|
from omniread.core.parser import BaseParser
|
||||||
|
|
||||||
|
T = TypeVar("T")
|
||||||
|
|
||||||
|
class HTMLParser(BaseParser[T], Generic[T]):
|
||||||
|
supported_types: set[ContentType]
|
||||||
|
def __init__(self, content: Content, features: str = ...) -> None: ...
|
||||||
|
def parse(self) -> T: ...
|
||||||
|
@staticmethod
|
||||||
|
def parse_div(div: Tag, *, separator: str = ...) -> str: ...
|
||||||
|
@staticmethod
|
||||||
|
def parse_link(a: Tag) -> str | None: ...
|
||||||
|
@staticmethod
|
||||||
|
def parse_table(table: Tag) -> list[list[str]]: ...
|
||||||
|
def parse_meta(self) -> dict[str, Any]: ...
|
||||||
@@ -1,4 +1,6 @@
|
|||||||
"""
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
HTML scraping implementation for OmniRead.
|
HTML scraping implementation for OmniRead.
|
||||||
|
|
||||||
This module provides an HTTP-based scraper for retrieving HTML documents.
|
This module provides an HTTP-based scraper for retrieving HTML documents.
|
||||||
@@ -6,18 +8,22 @@ It implements the core `BaseScraper` contract using `httpx` as the transport
|
|||||||
layer.
|
layer.
|
||||||
|
|
||||||
This scraper is responsible for:
|
This scraper is responsible for:
|
||||||
|
|
||||||
- Fetching raw HTML bytes over HTTP(S)
|
- Fetching raw HTML bytes over HTTP(S)
|
||||||
- Validating response content type
|
- Validating response content type
|
||||||
- Attaching HTTP metadata to the returned content
|
- Attaching HTTP metadata to the returned content
|
||||||
|
|
||||||
This scraper is not responsible for:
|
This scraper is not responsible for:
|
||||||
|
|
||||||
- Parsing or interpreting HTML
|
- Parsing or interpreting HTML
|
||||||
- Retrying failed requests
|
- Retrying failed requests
|
||||||
- Managing crawl policies or rate limiting
|
- Managing crawl policies or rate limiting
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from typing import Any, Mapping, Optional
|
|
||||||
|
|
||||||
from omniread.core.content import Content, ContentType
|
from omniread.core.content import Content, ContentType
|
||||||
from omniread.core.scraper import BaseScraper
|
from omniread.core.scraper import BaseScraper
|
||||||
@@ -25,21 +31,21 @@ from omniread.core.scraper import BaseScraper
|
|||||||
|
|
||||||
class HTMLScraper(BaseScraper):
|
class HTMLScraper(BaseScraper):
|
||||||
"""
|
"""
|
||||||
Base HTML scraper using httpx.
|
Base HTML scraper using `httpx`.
|
||||||
|
|
||||||
This scraper retrieves HTML documents over HTTP(S) and returns them
|
Notes:
|
||||||
as raw content wrapped in a `Content` object.
|
**Responsibilities:**
|
||||||
|
|
||||||
Fetches raw bytes and metadata only.
|
- This scraper retrieves HTML documents over HTTP(S) and returns
|
||||||
The scraper:
|
them as raw content wrapped in a `Content` object.
|
||||||
- Uses `httpx.Client` for HTTP requests
|
- Fetches raw bytes and metadata only.
|
||||||
- Enforces an HTML content type
|
- The scraper uses `httpx.Client` for HTTP requests, enforces an
|
||||||
- Preserves HTTP response metadata
|
HTML content type, and preserves HTTP response metadata.
|
||||||
|
|
||||||
The scraper does not:
|
**Constraints:**
|
||||||
- Parse HTML
|
|
||||||
- Perform retries or backoff
|
- The scraper does not: Parse HTML, perform retries or backoff,
|
||||||
- Handle non-HTML responses
|
handle non-HTML responses.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -47,18 +53,21 @@ class HTMLScraper(BaseScraper):
|
|||||||
*,
|
*,
|
||||||
client: httpx.Client | None = None,
|
client: httpx.Client | None = None,
|
||||||
timeout: float = 15.0,
|
timeout: float = 15.0,
|
||||||
headers: Optional[Mapping[str, str]] = None,
|
headers: Mapping[str, str] | None = None,
|
||||||
follow_redirects: bool = True,
|
follow_redirects: bool = True,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Initialize the HTML scraper.
|
Initialize the HTML scraper.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
client: Optional pre-configured `httpx.Client`. If omitted,
|
client (httpx.Client | None, optional):
|
||||||
a client is created internally.
|
Optional pre-configured `httpx.Client`. If omitted, a client is created internally.
|
||||||
timeout: Request timeout in seconds.
|
timeout (float, optional):
|
||||||
headers: Optional default HTTP headers.
|
Request timeout in seconds.
|
||||||
follow_redirects: Whether to follow HTTP redirects.
|
headers (Mapping[str, str] | None, optional):
|
||||||
|
Optional default HTTP headers.
|
||||||
|
follow_redirects (bool, optional):
|
||||||
|
Whether to follow HTTP redirects.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
self._client = client or httpx.Client(
|
self._client = client or httpx.Client(
|
||||||
@@ -76,11 +85,12 @@ class HTMLScraper(BaseScraper):
|
|||||||
Validate that the HTTP response contains HTML content.
|
Validate that the HTTP response contains HTML content.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
response: HTTP response returned by `httpx`.
|
response (httpx.Response):
|
||||||
|
HTTP response returned by `httpx`.
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
ValueError: If the `Content-Type` header is missing or does not
|
ValueError:
|
||||||
indicate HTML content.
|
If the `Content-Type` header is missing or does not indicate HTML content.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
raw_ct = response.headers.get("Content-Type")
|
raw_ct = response.headers.get("Content-Type")
|
||||||
@@ -89,33 +99,32 @@ class HTMLScraper(BaseScraper):
|
|||||||
|
|
||||||
base_ct = raw_ct.split(";", 1)[0].strip().lower()
|
base_ct = raw_ct.split(";", 1)[0].strip().lower()
|
||||||
if base_ct != self.content_type.value:
|
if base_ct != self.content_type.value:
|
||||||
raise ValueError(
|
raise ValueError(f"Expected HTML content, got '{raw_ct}'")
|
||||||
f"Expected HTML content, got '{raw_ct}'"
|
|
||||||
)
|
|
||||||
|
|
||||||
def fetch(
|
def fetch(
|
||||||
self,
|
self,
|
||||||
source: str,
|
source: str,
|
||||||
*,
|
*,
|
||||||
metadata: Optional[Mapping[str, Any]] = None,
|
metadata: Mapping[str, Any] | None = None,
|
||||||
) -> Content:
|
) -> Content:
|
||||||
"""
|
"""
|
||||||
Fetch an HTML document from the given source.
|
Fetch an HTML document from the given source.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
source: URL of the HTML document.
|
source (str):
|
||||||
metadata: Optional metadata to be merged into the returned content.
|
URL of the HTML document.
|
||||||
|
metadata (Mapping[str, Any] | None, optional):
|
||||||
|
Optional metadata to be merged into the returned content.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
A `Content` instance containing:
|
Content:
|
||||||
- Raw HTML bytes
|
A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.
|
||||||
- Source URL
|
|
||||||
- HTML content type
|
|
||||||
- HTTP response metadata
|
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
httpx.HTTPError: If the HTTP request fails.
|
httpx.HTTPError:
|
||||||
ValueError: If the response is not valid HTML.
|
If the HTTP request fails.
|
||||||
|
ValueError:
|
||||||
|
If the response is not valid HTML.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
response = self._client.get(source)
|
response = self._client.get(source)
|
||||||
|
|||||||
22
omniread/html/scraper.pyi
Normal file
22
omniread/html/scraper.pyi
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
from collections.abc import Mapping
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
from omniread.core.content import Content, ContentType
|
||||||
|
from omniread.core.scraper import BaseScraper
|
||||||
|
|
||||||
|
class HTMLScraper(BaseScraper):
|
||||||
|
content_type: ContentType
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
client: httpx.Client | None = ...,
|
||||||
|
timeout: float = ...,
|
||||||
|
headers: Mapping[str, str] | None = ...,
|
||||||
|
follow_redirects: bool = ...,
|
||||||
|
) -> None: ...
|
||||||
|
def validate_content_type(self, response: httpx.Response) -> None: ...
|
||||||
|
def fetch(
|
||||||
|
self, source: str, *, metadata: Mapping[str, Any] | None = ...
|
||||||
|
) -> Content: ...
|
||||||
@@ -1,4 +1,6 @@
|
|||||||
"""
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
PDF format implementation for OmniRead.
|
PDF format implementation for OmniRead.
|
||||||
|
|
||||||
This package provides **PDF-specific implementations** of the core OmniRead
|
This package provides **PDF-specific implementations** of the core OmniRead
|
||||||
@@ -6,12 +8,23 @@ contracts defined in `omniread.core`.
|
|||||||
|
|
||||||
Unlike HTML, PDF handling requires an explicit client layer for document
|
Unlike HTML, PDF handling requires an explicit client layer for document
|
||||||
access. This package therefore includes:
|
access. This package therefore includes:
|
||||||
- PDF clients for acquiring raw PDF data
|
|
||||||
- PDF scrapers that coordinate client access
|
- PDF clients for acquiring raw PDF data.
|
||||||
- PDF parsers that extract structured content from PDF binaries
|
- PDF scrapers that coordinate client access.
|
||||||
|
- PDF parsers that extract structured content from PDF binaries.
|
||||||
|
|
||||||
Public exports from this package represent the supported PDF pipeline
|
Public exports from this package represent the supported PDF pipeline
|
||||||
and are safe for consumers to import directly when working with PDFs.
|
and are safe for consumers to import directly when working with PDFs.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Public API
|
||||||
|
|
||||||
|
- `FileSystemPDFClient`
|
||||||
|
- `PDFScraper`
|
||||||
|
- `PDFParser`
|
||||||
|
|
||||||
|
---
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from .client import FileSystemPDFClient
|
from .client import FileSystemPDFClient
|
||||||
|
|||||||
5
omniread/pdf/__init__.pyi
Normal file
5
omniread/pdf/__init__.pyi
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
from .client import FileSystemPDFClient
|
||||||
|
from .parser import PDFParser
|
||||||
|
from .scraper import PDFScraper
|
||||||
|
|
||||||
|
__all__ = ["FileSystemPDFClient", "PDFScraper", "PDFParser"]
|
||||||
@@ -1,4 +1,6 @@
|
|||||||
"""
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
PDF client abstractions for OmniRead.
|
PDF client abstractions for OmniRead.
|
||||||
|
|
||||||
This module defines the **client layer** responsible for retrieving raw PDF
|
This module defines the **client layer** responsible for retrieving raw PDF
|
||||||
@@ -9,6 +11,7 @@ decoupled from scraping and parsing logic. They do not perform validation,
|
|||||||
interpretation, or content extraction.
|
interpretation, or content extraction.
|
||||||
|
|
||||||
Typical backing stores include:
|
Typical backing stores include:
|
||||||
|
|
||||||
- Local filesystems
|
- Local filesystems
|
||||||
- Object storage (S3, GCS, etc.)
|
- Object storage (S3, GCS, etc.)
|
||||||
- Network file systems
|
- Network file systems
|
||||||
@@ -16,33 +19,40 @@ Typical backing stores include:
|
|||||||
|
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
class BasePDFClient(ABC):
|
class BasePDFClient(ABC):
|
||||||
"""
|
"""
|
||||||
Abstract client responsible for retrieving PDF bytes
|
Abstract client responsible for retrieving PDF bytes.
|
||||||
from a specific backing store (filesystem, S3, FTP, etc.).
|
|
||||||
|
|
||||||
Implementations must:
|
Retrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).
|
||||||
- Accept a source identifier appropriate to the backing store
|
|
||||||
- Return the full PDF binary payload
|
Notes:
|
||||||
- Raise retrieval-specific errors on failure
|
**Responsibilities:**
|
||||||
|
|
||||||
|
- Implementations must accept a source identifier appropriate to
|
||||||
|
the backing store.
|
||||||
|
- Return the full PDF binary payload.
|
||||||
|
- Raise retrieval-specific errors on failure.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def fetch(self, source: str) -> bytes:
|
def fetch(self, source: Any) -> bytes:
|
||||||
"""
|
"""
|
||||||
Fetch raw PDF bytes from the given source.
|
Fetch raw PDF bytes from the given source.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
source: Identifier of the PDF location, such as a file path,
|
source (Any):
|
||||||
object storage key, or remote reference.
|
Identifier of the PDF location, such as a file path, object storage key, or remote reference.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
|
bytes:
|
||||||
Raw PDF bytes.
|
Raw PDF bytes.
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
Exception: Retrieval-specific errors defined by the implementation.
|
Exception:
|
||||||
|
Retrieval-specific errors defined by the implementation.
|
||||||
"""
|
"""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
@@ -51,8 +61,11 @@ class FileSystemPDFClient(BasePDFClient):
|
|||||||
"""
|
"""
|
||||||
PDF client that reads from the local filesystem.
|
PDF client that reads from the local filesystem.
|
||||||
|
|
||||||
This client reads PDF files directly from the disk and returns their raw
|
Notes:
|
||||||
binary contents.
|
**Guarantees:**
|
||||||
|
|
||||||
|
- This client reads PDF files directly from the disk and returns
|
||||||
|
their raw binary contents.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def fetch(self, path: Path) -> bytes:
|
def fetch(self, path: Path) -> bytes:
|
||||||
@@ -60,14 +73,18 @@ class FileSystemPDFClient(BasePDFClient):
|
|||||||
Read a PDF file from the local filesystem.
|
Read a PDF file from the local filesystem.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
path: Filesystem path to the PDF file.
|
path (Path):
|
||||||
|
Filesystem path to the PDF file.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
|
bytes:
|
||||||
Raw PDF bytes.
|
Raw PDF bytes.
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
FileNotFoundError: If the path does not exist.
|
FileNotFoundError:
|
||||||
ValueError: If the path exists but is not a file.
|
If the path does not exist.
|
||||||
|
ValueError:
|
||||||
|
If the path exists but is not a file.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
if not path.exists():
|
if not path.exists():
|
||||||
|
|||||||
10
omniread/pdf/client.pyi
Normal file
10
omniread/pdf/client.pyi
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
from abc import ABC, abstractmethod
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
class BasePDFClient(ABC):
|
||||||
|
@abstractmethod
|
||||||
|
def fetch(self, source: Any) -> bytes: ...
|
||||||
|
|
||||||
|
class FileSystemPDFClient(BasePDFClient):
|
||||||
|
def fetch(self, source: Path | str) -> bytes: ...
|
||||||
@@ -1,4 +1,6 @@
|
|||||||
"""
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
PDF parser base implementations for OmniRead.
|
PDF parser base implementations for OmniRead.
|
||||||
|
|
||||||
This module defines the **PDF-specific parser contract**, extending the
|
This module defines the **PDF-specific parser contract**, extending the
|
||||||
@@ -8,8 +10,8 @@ PDF parsers are responsible for interpreting binary PDF data and producing
|
|||||||
structured representations suitable for downstream consumption.
|
structured representations suitable for downstream consumption.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Generic, TypeVar
|
|
||||||
from abc import abstractmethod
|
from abc import abstractmethod
|
||||||
|
from typing import Generic, TypeVar
|
||||||
|
|
||||||
from omniread.core.content import ContentType
|
from omniread.core.content import ContentType
|
||||||
from omniread.core.parser import BaseParser
|
from omniread.core.parser import BaseParser
|
||||||
@@ -21,29 +23,40 @@ class PDFParser(BaseParser[T], Generic[T]):
|
|||||||
"""
|
"""
|
||||||
Base PDF parser.
|
Base PDF parser.
|
||||||
|
|
||||||
This class enforces PDF content-type compatibility and provides the
|
Notes:
|
||||||
extension point for implementing concrete PDF parsing strategies.
|
**Responsibilities:**
|
||||||
|
|
||||||
Concrete implementations must define:
|
- This class enforces PDF content-type compatibility and provides
|
||||||
- Define the output type `T`
|
the extension point for implementing concrete PDF parsing strategies.
|
||||||
- Implement the `parse()` method
|
|
||||||
|
**Constraints:**
|
||||||
|
|
||||||
|
- Concrete implementations must define the output type `T` and
|
||||||
|
implement the `parse()` method.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
supported_types = {ContentType.PDF}
|
supported_types = {ContentType.PDF}
|
||||||
"""Set of content types supported by this parser (PDF only)."""
|
"""
|
||||||
|
Set of content types supported by this parser (PDF only).
|
||||||
|
"""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def parse(self) -> T:
|
def parse(self) -> T:
|
||||||
"""
|
"""
|
||||||
Parse PDF content into a structured output.
|
Parse PDF content into a structured output.
|
||||||
|
|
||||||
Implementations must fully interpret the PDF binary payload and
|
|
||||||
return a deterministic, structured output.
|
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
|
T:
|
||||||
Parsed representation of type `T`.
|
Parsed representation of type `T`.
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
Exception: Parsing-specific errors as defined by the implementation.
|
Exception:
|
||||||
|
Parsing-specific errors as defined by the implementation.
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
**Responsibilities:**
|
||||||
|
|
||||||
|
- Implementations must fully interpret the PDF binary payload and
|
||||||
|
return a deterministic, structured output.
|
||||||
"""
|
"""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|||||||
12
omniread/pdf/parser.pyi
Normal file
12
omniread/pdf/parser.pyi
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
from abc import abstractmethod
|
||||||
|
from typing import Generic, TypeVar
|
||||||
|
|
||||||
|
from omniread.core.content import ContentType
|
||||||
|
from omniread.core.parser import BaseParser
|
||||||
|
|
||||||
|
T = TypeVar("T")
|
||||||
|
|
||||||
|
class PDFParser(BaseParser[T], Generic[T]):
|
||||||
|
supported_types: set[ContentType]
|
||||||
|
@abstractmethod
|
||||||
|
def parse(self) -> T: ...
|
||||||
@@ -1,4 +1,6 @@
|
|||||||
"""
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
PDF scraping implementation for OmniRead.
|
PDF scraping implementation for OmniRead.
|
||||||
|
|
||||||
This module provides a PDF-specific scraper that coordinates PDF byte
|
This module provides a PDF-specific scraper that coordinates PDF byte
|
||||||
@@ -8,7 +10,8 @@ The scraper implements the core `BaseScraper` contract while delegating
|
|||||||
all storage and access concerns to a `BasePDFClient` implementation.
|
all storage and access concerns to a `BasePDFClient` implementation.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Any, Mapping, Optional
|
from collections.abc import Mapping
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
from omniread.core.content import Content, ContentType
|
from omniread.core.content import Content, ContentType
|
||||||
from omniread.core.scraper import BaseScraper
|
from omniread.core.scraper import BaseScraper
|
||||||
@@ -19,13 +22,17 @@ class PDFScraper(BaseScraper):
|
|||||||
"""
|
"""
|
||||||
Scraper for PDF sources.
|
Scraper for PDF sources.
|
||||||
|
|
||||||
Delegates byte retrieval to a PDF client and normalizes
|
Notes:
|
||||||
output into Content.
|
**Responsibilities:**
|
||||||
|
|
||||||
The scraper:
|
- Delegates byte retrieval to a PDF client and normalizes output
|
||||||
- Does not perform parsing or interpretation
|
into `Content`.
|
||||||
- Does not assume a specific storage backend
|
- Preserves caller-provided metadata.
|
||||||
- Preserves caller-provided metadata
|
|
||||||
|
**Constraints:**
|
||||||
|
|
||||||
|
- The scraper does not perform parsing or interpretation.
|
||||||
|
- Does not assume a specific storage backend.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, *, client: BasePDFClient):
|
def __init__(self, *, client: BasePDFClient):
|
||||||
@@ -33,33 +40,33 @@ class PDFScraper(BaseScraper):
|
|||||||
Initialize the PDF scraper.
|
Initialize the PDF scraper.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
client: PDF client responsible for retrieving raw PDF bytes.
|
client (BasePDFClient):
|
||||||
|
PDF client responsible for retrieving raw PDF bytes.
|
||||||
"""
|
"""
|
||||||
self._client = client
|
self._client = client
|
||||||
|
|
||||||
def fetch(
|
def fetch(
|
||||||
self,
|
self,
|
||||||
source: str,
|
source: Any,
|
||||||
*,
|
*,
|
||||||
metadata: Optional[Mapping[str, Any]] = None,
|
metadata: Mapping[str, Any] | None = None,
|
||||||
) -> Content:
|
) -> Content:
|
||||||
"""
|
"""
|
||||||
Fetch a PDF document from the given source.
|
Fetch a PDF document from the given source.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
source: Identifier of the PDF source as understood by the
|
source (Any):
|
||||||
configured PDF client.
|
Identifier of the PDF source as understood by the configured PDF client.
|
||||||
metadata: Optional metadata to attach to the returned content.
|
metadata (Mapping[str, Any] | None, optional):
|
||||||
|
Optional metadata to attach to the returned content.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
A `Content` instance containing:
|
Content:
|
||||||
- Raw PDF bytes
|
A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.
|
||||||
- Source identifier
|
|
||||||
- PDF content type
|
|
||||||
- Optional metadata
|
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
Exception: Retrieval-specific errors raised by the PDF client.
|
Exception:
|
||||||
|
Retrieval-specific errors raised by the PDF client.
|
||||||
"""
|
"""
|
||||||
raw = self._client.fetch(source)
|
raw = self._client.fetch(source)
|
||||||
|
|
||||||
|
|||||||
13
omniread/pdf/scraper.pyi
Normal file
13
omniread/pdf/scraper.pyi
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
from collections.abc import Mapping
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from omniread.core.content import Content
|
||||||
|
from omniread.core.scraper import BaseScraper
|
||||||
|
|
||||||
|
from .client import BasePDFClient
|
||||||
|
|
||||||
|
class PDFScraper(BaseScraper):
|
||||||
|
def __init__(self, *, client: BasePDFClient) -> None: ...
|
||||||
|
def fetch(
|
||||||
|
self, source: Any, *, metadata: Mapping[str, Any] | None = ...
|
||||||
|
) -> Content: ...
|
||||||
0
omniread/py.typed
Normal file
0
omniread/py.typed
Normal file
27
omniread/xlsx/__init__.py
Normal file
27
omniread/xlsx/__init__.py
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
|
XLSX subpackage for OmniRead.
|
||||||
|
|
||||||
|
Provides acquisition and parsing of Office Open XML spreadsheet (xlsx)
|
||||||
|
content:
|
||||||
|
|
||||||
|
- `BaseXlsxClient`: abstract backing-store client for xlsx bytes.
|
||||||
|
- `FileSystemXlsxClient`: local filesystem implementation.
|
||||||
|
- `XlsxScraper`: wraps fetched bytes into canonical `Content`.
|
||||||
|
- `XlsxParserBase`: content-type-enforcing parser contract.
|
||||||
|
- `XlsxParser`: generic string-row parser built on openpyxl.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from .client import BaseXlsxClient, FileSystemXlsxClient
|
||||||
|
from .parser import XlsxParser
|
||||||
|
from .parser_base import XlsxParserBase
|
||||||
|
from .scraper import XlsxScraper
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"BaseXlsxClient",
|
||||||
|
"FileSystemXlsxClient",
|
||||||
|
"XlsxScraper",
|
||||||
|
"XlsxParser",
|
||||||
|
"XlsxParserBase",
|
||||||
|
]
|
||||||
12
omniread/xlsx/__init__.pyi
Normal file
12
omniread/xlsx/__init__.pyi
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
from .client import BaseXlsxClient, FileSystemXlsxClient
|
||||||
|
from .parser import XlsxParser
|
||||||
|
from .parser_base import XlsxParserBase
|
||||||
|
from .scraper import XlsxScraper
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"BaseXlsxClient",
|
||||||
|
"FileSystemXlsxClient",
|
||||||
|
"XlsxScraper",
|
||||||
|
"XlsxParser",
|
||||||
|
"XlsxParserBase",
|
||||||
|
]
|
||||||
97
omniread/xlsx/client.py
Normal file
97
omniread/xlsx/client.py
Normal file
@@ -0,0 +1,97 @@
|
|||||||
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
|
XLSX client abstractions for OmniRead.
|
||||||
|
|
||||||
|
This module defines the **client layer** responsible for retrieving raw
|
||||||
|
Office Open XML spreadsheet bytes from a concrete backing store.
|
||||||
|
|
||||||
|
Clients provide low-level access to xlsx binaries and are intentionally
|
||||||
|
decoupled from scraping and parsing logic. They do not perform validation,
|
||||||
|
interpretation, or content extraction.
|
||||||
|
|
||||||
|
Typical backing stores include:
|
||||||
|
|
||||||
|
- Local filesystems
|
||||||
|
- Object storage (S3, GCS, etc.)
|
||||||
|
- Network file systems
|
||||||
|
"""
|
||||||
|
|
||||||
|
from abc import ABC, abstractmethod
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
class BaseXlsxClient(ABC):
|
||||||
|
"""
|
||||||
|
Abstract client responsible for retrieving spreadsheet bytes.
|
||||||
|
|
||||||
|
Retrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
**Responsibilities:**
|
||||||
|
|
||||||
|
- Implementations must accept a source identifier appropriate to
|
||||||
|
the backing store.
|
||||||
|
- Return the full xlsx binary payload.
|
||||||
|
- Raise retrieval-specific errors on failure.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def fetch(self, source: Any) -> bytes:
|
||||||
|
"""
|
||||||
|
Fetch raw xlsx bytes from the given source.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
source (Any):
|
||||||
|
Identifier of the spreadsheet location, such as a file path,
|
||||||
|
object storage key, or remote reference.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
bytes:
|
||||||
|
Raw xlsx bytes.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
Exception:
|
||||||
|
Retrieval-specific errors defined by the implementation.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
|
||||||
|
class FileSystemXlsxClient(BaseXlsxClient):
|
||||||
|
"""
|
||||||
|
XLSX client that reads from the local filesystem.
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
**Guarantees:**
|
||||||
|
|
||||||
|
- This client reads spreadsheet files directly from the disk and
|
||||||
|
returns their raw binary contents.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def fetch(self, path: Path) -> bytes:
|
||||||
|
"""
|
||||||
|
Read an xlsx file from the local filesystem.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
path (Path):
|
||||||
|
Filesystem path to the spreadsheet file.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
bytes:
|
||||||
|
Raw xlsx bytes.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
FileNotFoundError:
|
||||||
|
If the path does not exist.
|
||||||
|
ValueError:
|
||||||
|
If the path exists but is not a file.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if not path.exists():
|
||||||
|
raise FileNotFoundError(f"XLSX not found: {path}")
|
||||||
|
|
||||||
|
if not path.is_file():
|
||||||
|
raise ValueError(f"Path is not a file: {path}")
|
||||||
|
|
||||||
|
return path.read_bytes()
|
||||||
10
omniread/xlsx/client.pyi
Normal file
10
omniread/xlsx/client.pyi
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
from abc import ABC, abstractmethod
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
class BaseXlsxClient(ABC):
|
||||||
|
@abstractmethod
|
||||||
|
def fetch(self, source: Any) -> bytes: ...
|
||||||
|
|
||||||
|
class FileSystemXlsxClient(BaseXlsxClient):
|
||||||
|
def fetch(self, path: Path) -> bytes: ...
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user