Compare commits
5 Commits
0.0.2
...
cd06de5435
| Author | SHA1 | Date | |
|---|---|---|---|
| cd06de5435 | |||
| 14d89f5f91 | |||
| 7283a3eafd | |||
| 3ebfa34733 | |||
| 1e7e7c8a6c |
22
.drone.yml
22
.drone.yml
@@ -32,6 +32,26 @@ steps:
|
|||||||
echo "🆕 New version detected: $PACKAGE_NAME==$VERSION"
|
echo "🆕 New version detected: $PACKAGE_NAME==$VERSION"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
- name: quality-gate
|
||||||
|
image: python:3.13-slim
|
||||||
|
environment:
|
||||||
|
PIP_REPO_URL:
|
||||||
|
from_secret: PIP_REPO_URL
|
||||||
|
PIP_USERNAME:
|
||||||
|
from_secret: PIP_USERNAME
|
||||||
|
PIP_PASSWORD:
|
||||||
|
from_secret: PIP_PASSWORD
|
||||||
|
commands:
|
||||||
|
- pip install --upgrade pip build
|
||||||
|
- |
|
||||||
|
AUTH_URL="https://${PIP_USERNAME}:${PIP_PASSWORD}@$(echo "${PIP_REPO_URL#*://}" | sed 's:/*$::')/simple"
|
||||||
|
pip install --index-url "$AUTH_URL" --extra-index-url https://pypi.org/simple/ -U ".[dev]"
|
||||||
|
- echo "🛡️ Running quality gate..."
|
||||||
|
- python -m black --check .
|
||||||
|
- python -m ruff check .
|
||||||
|
- python -m mypy
|
||||||
|
- python -m pytest
|
||||||
|
|
||||||
- name: build-package
|
- name: build-package
|
||||||
image: python:3.13-slim
|
image: python:3.13-slim
|
||||||
commands:
|
commands:
|
||||||
@@ -126,4 +146,4 @@ steps:
|
|||||||
|
|
||||||
trigger:
|
trigger:
|
||||||
event:
|
event:
|
||||||
- custom
|
- custom
|
||||||
21
CHANGELOG.md
Normal file
21
CHANGELOG.md
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
# Changelog
|
||||||
|
|
||||||
|
All notable changes to this project will be documented in this file.
|
||||||
|
|
||||||
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||||
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||||
|
|
||||||
|
## [Unreleased]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- `py.typed` marker for PEP 561 type information.
|
||||||
|
- `.drone.yml` CI with a quality-gate step (black, ruff, mypy, pytest).
|
||||||
|
- Canonical `docforge.nav.yml`, generated `mkdocs.yml` and `mcp_docs/` via doc-forge.
|
||||||
|
- MIT `LICENSE`.
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
- Standardized `pyproject.toml` (canonical packaging, lint tool config, extras).
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
- `XLSX`/`CSV` members added to the `content` type surface.
|
||||||
|
- Stub fixes for typed API surfaces.
|
||||||
21
LICENSE
Normal file
21
LICENSE
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2026 Aetoskia Platform
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
@@ -1,16 +1,19 @@
|
|||||||
home: index.md
|
home: lib/index.md
|
||||||
groups:
|
groups:
|
||||||
Core API:
|
Core API:
|
||||||
- core/index.md
|
- lib/core/index.md
|
||||||
- core/content.md
|
- lib/core/content.md
|
||||||
- core/parser.md
|
- lib/core/parser.md
|
||||||
- core/scraper.md
|
- lib/core/scraper.md
|
||||||
HTML Handling:
|
HTML Handling:
|
||||||
- html/index.md
|
- lib/html/index.md
|
||||||
- html/parser.md
|
- lib/html/parser.md
|
||||||
- html/scraper.md
|
- lib/html/scraper.md
|
||||||
PDF Handling:
|
PDF Handling:
|
||||||
- pdf/index.md
|
- lib/pdf/index.md
|
||||||
- pdf/client.md
|
- lib/pdf/client.md
|
||||||
- pdf/parser.md
|
- lib/pdf/parser.md
|
||||||
- pdf/scraper.md
|
- lib/pdf/scraper.md
|
||||||
|
icon:
|
||||||
|
logo: material/book-open-page-variant
|
||||||
|
repo: fontawesome/brands/github
|
||||||
3
docs/lib/csv/client.md
Normal file
3
docs/lib/csv/client.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Client
|
||||||
|
|
||||||
|
::: omniread.csv.client
|
||||||
3
docs/lib/csv/index.md
Normal file
3
docs/lib/csv/index.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Csv
|
||||||
|
|
||||||
|
::: omniread.csv
|
||||||
3
docs/lib/csv/parser.md
Normal file
3
docs/lib/csv/parser.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Parser
|
||||||
|
|
||||||
|
::: omniread.csv.parser
|
||||||
3
docs/lib/csv/parser_base.md
Normal file
3
docs/lib/csv/parser_base.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Parser Base
|
||||||
|
|
||||||
|
::: omniread.csv.parser_base
|
||||||
3
docs/lib/csv/scraper.md
Normal file
3
docs/lib/csv/scraper.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Scraper
|
||||||
|
|
||||||
|
::: omniread.csv.scraper
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
# omniread
|
# omniread
|
||||||
|
|
||||||
::: omniread
|
::: omniread
|
||||||
|
- [Omniread](omniread/)
|
||||||
3
docs/lib/omniread/core/content.md
Normal file
3
docs/lib/omniread/core/content.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Content
|
||||||
|
|
||||||
|
::: omniread.core.content
|
||||||
6
docs/lib/omniread/core/index.md
Normal file
6
docs/lib/omniread/core/index.md
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
# Core
|
||||||
|
|
||||||
|
::: omniread.core
|
||||||
|
- [Content](content.md)
|
||||||
|
- [Parser](parser.md)
|
||||||
|
- [Scraper](scraper.md)
|
||||||
3
docs/lib/omniread/core/parser.md
Normal file
3
docs/lib/omniread/core/parser.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Parser
|
||||||
|
|
||||||
|
::: omniread.core.parser
|
||||||
3
docs/lib/omniread/core/scraper.md
Normal file
3
docs/lib/omniread/core/scraper.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Scraper
|
||||||
|
|
||||||
|
::: omniread.core.scraper
|
||||||
3
docs/lib/omniread/csv/client.md
Normal file
3
docs/lib/omniread/csv/client.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Client
|
||||||
|
|
||||||
|
::: omniread.csv.client
|
||||||
7
docs/lib/omniread/csv/index.md
Normal file
7
docs/lib/omniread/csv/index.md
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
# Csv
|
||||||
|
|
||||||
|
::: omniread.csv
|
||||||
|
- [Client](client.md)
|
||||||
|
- [Parser](parser.md)
|
||||||
|
- [Parser Base](parser_base.md)
|
||||||
|
- [Scraper](scraper.md)
|
||||||
3
docs/lib/omniread/csv/parser.md
Normal file
3
docs/lib/omniread/csv/parser.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Parser
|
||||||
|
|
||||||
|
::: omniread.csv.parser
|
||||||
3
docs/lib/omniread/csv/parser_base.md
Normal file
3
docs/lib/omniread/csv/parser_base.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Parser Base
|
||||||
|
|
||||||
|
::: omniread.csv.parser_base
|
||||||
3
docs/lib/omniread/csv/scraper.md
Normal file
3
docs/lib/omniread/csv/scraper.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Scraper
|
||||||
|
|
||||||
|
::: omniread.csv.scraper
|
||||||
5
docs/lib/omniread/html/index.md
Normal file
5
docs/lib/omniread/html/index.md
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
# Html
|
||||||
|
|
||||||
|
::: omniread.html
|
||||||
|
- [Parser](parser.md)
|
||||||
|
- [Scraper](scraper.md)
|
||||||
3
docs/lib/omniread/html/parser.md
Normal file
3
docs/lib/omniread/html/parser.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Parser
|
||||||
|
|
||||||
|
::: omniread.html.parser
|
||||||
3
docs/lib/omniread/html/scraper.md
Normal file
3
docs/lib/omniread/html/scraper.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Scraper
|
||||||
|
|
||||||
|
::: omniread.html.scraper
|
||||||
8
docs/lib/omniread/index.md
Normal file
8
docs/lib/omniread/index.md
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
# Omniread
|
||||||
|
|
||||||
|
::: omniread
|
||||||
|
- [Core](core/)
|
||||||
|
- [Csv](csv/)
|
||||||
|
- [Html](html/)
|
||||||
|
- [Pdf](pdf/)
|
||||||
|
- [Xlsx](xlsx/)
|
||||||
6
docs/lib/omniread/pdf/index.md
Normal file
6
docs/lib/omniread/pdf/index.md
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
# Pdf
|
||||||
|
|
||||||
|
::: omniread.pdf
|
||||||
|
- [Client](client.md)
|
||||||
|
- [Parser](parser.md)
|
||||||
|
- [Scraper](scraper.md)
|
||||||
3
docs/lib/omniread/xlsx/client.md
Normal file
3
docs/lib/omniread/xlsx/client.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Client
|
||||||
|
|
||||||
|
::: omniread.xlsx.client
|
||||||
7
docs/lib/omniread/xlsx/index.md
Normal file
7
docs/lib/omniread/xlsx/index.md
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
# Xlsx
|
||||||
|
|
||||||
|
::: omniread.xlsx
|
||||||
|
- [Client](client.md)
|
||||||
|
- [Parser](parser.md)
|
||||||
|
- [Parser Base](parser_base.md)
|
||||||
|
- [Scraper](scraper.md)
|
||||||
3
docs/lib/omniread/xlsx/parser.md
Normal file
3
docs/lib/omniread/xlsx/parser.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Parser
|
||||||
|
|
||||||
|
::: omniread.xlsx.parser
|
||||||
3
docs/lib/omniread/xlsx/parser_base.md
Normal file
3
docs/lib/omniread/xlsx/parser_base.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Parser Base
|
||||||
|
|
||||||
|
::: omniread.xlsx.parser_base
|
||||||
3
docs/lib/omniread/xlsx/scraper.md
Normal file
3
docs/lib/omniread/xlsx/scraper.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Scraper
|
||||||
|
|
||||||
|
::: omniread.xlsx.scraper
|
||||||
3
docs/lib/pdf/client.md
Normal file
3
docs/lib/pdf/client.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Client
|
||||||
|
|
||||||
|
::: omniread.pdf.client
|
||||||
3
docs/lib/pdf/parser.md
Normal file
3
docs/lib/pdf/parser.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Parser
|
||||||
|
|
||||||
|
::: omniread.pdf.parser
|
||||||
3
docs/lib/pdf/scraper.md
Normal file
3
docs/lib/pdf/scraper.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Scraper
|
||||||
|
|
||||||
|
::: omniread.pdf.scraper
|
||||||
3
docs/lib/xlsx/client.md
Normal file
3
docs/lib/xlsx/client.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Client
|
||||||
|
|
||||||
|
::: omniread.xlsx.client
|
||||||
3
docs/lib/xlsx/index.md
Normal file
3
docs/lib/xlsx/index.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Xlsx
|
||||||
|
|
||||||
|
::: omniread.xlsx
|
||||||
3
docs/lib/xlsx/parser.md
Normal file
3
docs/lib/xlsx/parser.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Parser
|
||||||
|
|
||||||
|
::: omniread.xlsx.parser
|
||||||
3
docs/lib/xlsx/parser_base.md
Normal file
3
docs/lib/xlsx/parser_base.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Parser Base
|
||||||
|
|
||||||
|
::: omniread.xlsx.parser_base
|
||||||
3
docs/lib/xlsx/scraper.md
Normal file
3
docs/lib/xlsx/scraper.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# Scraper
|
||||||
|
|
||||||
|
::: omniread.xlsx.scraper
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"project": "omniread",
|
"project": "omniread",
|
||||||
"type": "docforge-model",
|
"type": "docforge-model",
|
||||||
"modules_count": 12,
|
"modules_count": 22,
|
||||||
"source": "docforge"
|
"source": "docforge"
|
||||||
}
|
}
|
||||||
@@ -4,11 +4,11 @@
|
|||||||
"path": "omniread.core.content",
|
"path": "omniread.core.content",
|
||||||
"docstring": "# Summary\n\nCanonical content models for OmniRead.\n\nThis module defines the **format-agnostic content representation** used across\nall parsers and scrapers in OmniRead.\n\nThe models defined here represent *what* was extracted, not *how* it was\nretrieved or parsed. Format-specific behavior and metadata must not alter\nthe semantic meaning of these models.",
|
"docstring": "# Summary\n\nCanonical content models for OmniRead.\n\nThis module defines the **format-agnostic content representation** used across\nall parsers and scrapers in OmniRead.\n\nThe models defined here represent *what* was extracted, not *how* it was\nretrieved or parsed. Format-specific behavior and metadata must not alter\nthe semantic meaning of these models.",
|
||||||
"objects": {
|
"objects": {
|
||||||
"Enum": {
|
"Mapping": {
|
||||||
"name": "Enum",
|
"name": "Mapping",
|
||||||
"kind": "alias",
|
"kind": "alias",
|
||||||
"path": "omniread.core.content.Enum",
|
"path": "omniread.core.content.Mapping",
|
||||||
"signature": "<bound method Alias.signature of Alias('Enum', 'enum.Enum')>",
|
"signature": "<bound method Alias.signature of Alias('Mapping', 'collections.abc.Mapping')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
"dataclass": {
|
"dataclass": {
|
||||||
@@ -18,6 +18,13 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('dataclass', 'dataclasses.dataclass')>",
|
"signature": "<bound method Alias.signature of Alias('dataclass', 'dataclasses.dataclass')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
|
"Enum": {
|
||||||
|
"name": "Enum",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.core.content.Enum",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Enum', 'enum.Enum')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
"Any": {
|
"Any": {
|
||||||
"name": "Any",
|
"name": "Any",
|
||||||
"kind": "alias",
|
"kind": "alias",
|
||||||
@@ -25,25 +32,11 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
"Mapping": {
|
|
||||||
"name": "Mapping",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.core.content.Mapping",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('Mapping', 'typing.Mapping')>",
|
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"Optional": {
|
|
||||||
"name": "Optional",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.core.content.Optional",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('Optional', 'typing.Optional')>",
|
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"ContentType": {
|
"ContentType": {
|
||||||
"name": "ContentType",
|
"name": "ContentType",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
"path": "omniread.core.content.ContentType",
|
"path": "omniread.core.content.ContentType",
|
||||||
"signature": "<bound method Class.signature of Class('ContentType', 19, 42)>",
|
"signature": "<bound method Class.signature of Class('ContentType', 20, 49)>",
|
||||||
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
"members": {
|
"members": {
|
||||||
"HTML": {
|
"HTML": {
|
||||||
@@ -60,6 +53,20 @@
|
|||||||
"signature": null,
|
"signature": null,
|
||||||
"docstring": "PDF document content."
|
"docstring": "PDF document content."
|
||||||
},
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.content.ContentType.XLSX",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.content.ContentType.CSV",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
"JSON": {
|
"JSON": {
|
||||||
"name": "JSON",
|
"name": "JSON",
|
||||||
"kind": "attribute",
|
"kind": "attribute",
|
||||||
@@ -80,7 +87,7 @@
|
|||||||
"name": "Content",
|
"name": "Content",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
"path": "omniread.core.content.Content",
|
"path": "omniread.core.content.Content",
|
||||||
"signature": "<bound method Class.signature of Class('Content', 45, 77)>",
|
"signature": "<bound method Class.signature of Class('Content', 52, 84)>",
|
||||||
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
"members": {
|
"members": {
|
||||||
"raw": {
|
"raw": {
|
||||||
@@ -62,6 +62,20 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
||||||
"docstring": "PDF document content."
|
"docstring": "PDF document content."
|
||||||
},
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.ContentType.XLSX",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XLSX', 'omniread.core.content.ContentType.XLSX')>",
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.ContentType.CSV",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('CSV', 'omniread.core.content.ContentType.CSV')>",
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
"JSON": {
|
"JSON": {
|
||||||
"name": "JSON",
|
"name": "JSON",
|
||||||
"kind": "attribute",
|
"kind": "attribute",
|
||||||
@@ -127,7 +141,7 @@
|
|||||||
"kind": "function",
|
"kind": "function",
|
||||||
"path": "omniread.core.BaseScraper.fetch",
|
"path": "omniread.core.BaseScraper.fetch",
|
||||||
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.core.scraper.BaseScraper.fetch')>",
|
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.core.scraper.BaseScraper.fetch')>",
|
||||||
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
|
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -138,11 +152,11 @@
|
|||||||
"signature": null,
|
"signature": null,
|
||||||
"docstring": "# Summary\n\nCanonical content models for OmniRead.\n\nThis module defines the **format-agnostic content representation** used across\nall parsers and scrapers in OmniRead.\n\nThe models defined here represent *what* was extracted, not *how* it was\nretrieved or parsed. Format-specific behavior and metadata must not alter\nthe semantic meaning of these models.",
|
"docstring": "# Summary\n\nCanonical content models for OmniRead.\n\nThis module defines the **format-agnostic content representation** used across\nall parsers and scrapers in OmniRead.\n\nThe models defined here represent *what* was extracted, not *how* it was\nretrieved or parsed. Format-specific behavior and metadata must not alter\nthe semantic meaning of these models.",
|
||||||
"members": {
|
"members": {
|
||||||
"Enum": {
|
"Mapping": {
|
||||||
"name": "Enum",
|
"name": "Mapping",
|
||||||
"kind": "alias",
|
"kind": "alias",
|
||||||
"path": "omniread.core.content.Enum",
|
"path": "omniread.core.content.Mapping",
|
||||||
"signature": "<bound method Alias.signature of Alias('Enum', 'enum.Enum')>",
|
"signature": "<bound method Alias.signature of Alias('Mapping', 'collections.abc.Mapping')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
"dataclass": {
|
"dataclass": {
|
||||||
@@ -152,6 +166,13 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('dataclass', 'dataclasses.dataclass')>",
|
"signature": "<bound method Alias.signature of Alias('dataclass', 'dataclasses.dataclass')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
|
"Enum": {
|
||||||
|
"name": "Enum",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.core.content.Enum",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Enum', 'enum.Enum')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
"Any": {
|
"Any": {
|
||||||
"name": "Any",
|
"name": "Any",
|
||||||
"kind": "alias",
|
"kind": "alias",
|
||||||
@@ -159,25 +180,11 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
"Mapping": {
|
|
||||||
"name": "Mapping",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.core.content.Mapping",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('Mapping', 'typing.Mapping')>",
|
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"Optional": {
|
|
||||||
"name": "Optional",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.core.content.Optional",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('Optional', 'typing.Optional')>",
|
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"ContentType": {
|
"ContentType": {
|
||||||
"name": "ContentType",
|
"name": "ContentType",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
"path": "omniread.core.content.ContentType",
|
"path": "omniread.core.content.ContentType",
|
||||||
"signature": "<bound method Class.signature of Class('ContentType', 19, 42)>",
|
"signature": "<bound method Class.signature of Class('ContentType', 20, 49)>",
|
||||||
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
"members": {
|
"members": {
|
||||||
"HTML": {
|
"HTML": {
|
||||||
@@ -194,6 +201,20 @@
|
|||||||
"signature": null,
|
"signature": null,
|
||||||
"docstring": "PDF document content."
|
"docstring": "PDF document content."
|
||||||
},
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.content.ContentType.XLSX",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.content.ContentType.CSV",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
"JSON": {
|
"JSON": {
|
||||||
"name": "JSON",
|
"name": "JSON",
|
||||||
"kind": "attribute",
|
"kind": "attribute",
|
||||||
@@ -214,7 +235,7 @@
|
|||||||
"name": "Content",
|
"name": "Content",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
"path": "omniread.core.content.Content",
|
"path": "omniread.core.content.Content",
|
||||||
"signature": "<bound method Class.signature of Class('Content', 45, 77)>",
|
"signature": "<bound method Class.signature of Class('Content', 52, 84)>",
|
||||||
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
"members": {
|
"members": {
|
||||||
"raw": {
|
"raw": {
|
||||||
@@ -284,13 +305,6 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('TypeVar', 'typing.TypeVar')>",
|
"signature": "<bound method Alias.signature of Alias('TypeVar', 'typing.TypeVar')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
"Set": {
|
|
||||||
"name": "Set",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.core.parser.Set",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('Set', 'typing.Set')>",
|
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"Content": {
|
"Content": {
|
||||||
"name": "Content",
|
"name": "Content",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
@@ -349,6 +363,20 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
||||||
"docstring": "PDF document content."
|
"docstring": "PDF document content."
|
||||||
},
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.ContentType.XLSX",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XLSX', 'omniread.core.content.ContentType.XLSX')>",
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.ContentType.CSV",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('CSV', 'omniread.core.content.ContentType.CSV')>",
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
"JSON": {
|
"JSON": {
|
||||||
"name": "JSON",
|
"name": "JSON",
|
||||||
"kind": "attribute",
|
"kind": "attribute",
|
||||||
@@ -432,6 +460,13 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
|
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
|
"Mapping": {
|
||||||
|
"name": "Mapping",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.core.scraper.Mapping",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Mapping', 'collections.abc.Mapping')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
"Any": {
|
"Any": {
|
||||||
"name": "Any",
|
"name": "Any",
|
||||||
"kind": "alias",
|
"kind": "alias",
|
||||||
@@ -439,20 +474,6 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
"Mapping": {
|
|
||||||
"name": "Mapping",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.core.scraper.Mapping",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('Mapping', 'typing.Mapping')>",
|
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"Optional": {
|
|
||||||
"name": "Optional",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.core.scraper.Optional",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('Optional', 'typing.Optional')>",
|
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"Content": {
|
"Content": {
|
||||||
"name": "Content",
|
"name": "Content",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
@@ -494,15 +515,15 @@
|
|||||||
"name": "BaseScraper",
|
"name": "BaseScraper",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
"path": "omniread.core.scraper.BaseScraper",
|
"path": "omniread.core.scraper.BaseScraper",
|
||||||
"signature": "<bound method Class.signature of Class('BaseScraper', 30, 82)>",
|
"signature": "<bound method Class.signature of Class('BaseScraper', 31, 83)>",
|
||||||
"docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.",
|
"docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.",
|
||||||
"members": {
|
"members": {
|
||||||
"fetch": {
|
"fetch": {
|
||||||
"name": "fetch",
|
"name": "fetch",
|
||||||
"kind": "function",
|
"kind": "function",
|
||||||
"path": "omniread.core.scraper.BaseScraper.fetch",
|
"path": "omniread.core.scraper.BaseScraper.fetch",
|
||||||
"signature": "<bound method Function.signature of Function('fetch', 51, 82)>",
|
"signature": "<bound method Function.signature of Function('fetch', 52, 83)>",
|
||||||
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
|
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -32,13 +32,6 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('TypeVar', 'typing.TypeVar')>",
|
"signature": "<bound method Alias.signature of Alias('TypeVar', 'typing.TypeVar')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
"Set": {
|
|
||||||
"name": "Set",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.core.parser.Set",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('Set', 'typing.Set')>",
|
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"Content": {
|
"Content": {
|
||||||
"name": "Content",
|
"name": "Content",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
@@ -97,6 +90,20 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
||||||
"docstring": "PDF document content."
|
"docstring": "PDF document content."
|
||||||
},
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.ContentType.XLSX",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XLSX', 'omniread.core.content.ContentType.XLSX')>",
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.core.parser.ContentType.CSV",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('CSV', 'omniread.core.content.ContentType.CSV')>",
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
"JSON": {
|
"JSON": {
|
||||||
"name": "JSON",
|
"name": "JSON",
|
||||||
"kind": "attribute",
|
"kind": "attribute",
|
||||||
@@ -18,6 +18,13 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
|
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
|
"Mapping": {
|
||||||
|
"name": "Mapping",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.core.scraper.Mapping",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Mapping', 'collections.abc.Mapping')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
"Any": {
|
"Any": {
|
||||||
"name": "Any",
|
"name": "Any",
|
||||||
"kind": "alias",
|
"kind": "alias",
|
||||||
@@ -25,20 +32,6 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
"Mapping": {
|
|
||||||
"name": "Mapping",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.core.scraper.Mapping",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('Mapping', 'typing.Mapping')>",
|
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"Optional": {
|
|
||||||
"name": "Optional",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.core.scraper.Optional",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('Optional', 'typing.Optional')>",
|
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"Content": {
|
"Content": {
|
||||||
"name": "Content",
|
"name": "Content",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
@@ -80,15 +73,15 @@
|
|||||||
"name": "BaseScraper",
|
"name": "BaseScraper",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
"path": "omniread.core.scraper.BaseScraper",
|
"path": "omniread.core.scraper.BaseScraper",
|
||||||
"signature": "<bound method Class.signature of Class('BaseScraper', 30, 82)>",
|
"signature": "<bound method Class.signature of Class('BaseScraper', 31, 83)>",
|
||||||
"docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.",
|
"docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.",
|
||||||
"members": {
|
"members": {
|
||||||
"fetch": {
|
"fetch": {
|
||||||
"name": "fetch",
|
"name": "fetch",
|
||||||
"kind": "function",
|
"kind": "function",
|
||||||
"path": "omniread.core.scraper.BaseScraper.fetch",
|
"path": "omniread.core.scraper.BaseScraper.fetch",
|
||||||
"signature": "<bound method Function.signature of Function('fetch', 51, 82)>",
|
"signature": "<bound method Function.signature of Function('fetch', 52, 83)>",
|
||||||
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
|
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
69
docs/mcp/modules/omniread.csv.client.json
Normal file
69
docs/mcp/modules/omniread.csv.client.json
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.csv.client",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.csv.client",
|
||||||
|
"docstring": "# Summary\n\nCSV client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw\ncomma-separated-value document bytes from a concrete backing store.\n\nClients provide low-level access to csv binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems",
|
||||||
|
"objects": {
|
||||||
|
"ABC": {
|
||||||
|
"name": "ABC",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.csv.client.ABC",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('ABC', 'abc.ABC')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"abstractmethod": {
|
||||||
|
"name": "abstractmethod",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.csv.client.abstractmethod",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Path": {
|
||||||
|
"name": "Path",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.csv.client.Path",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Path', 'pathlib.Path')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Any": {
|
||||||
|
"name": "Any",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.csv.client.Any",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"BaseCsvClient": {
|
||||||
|
"name": "BaseCsvClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.client.BaseCsvClient",
|
||||||
|
"signature": "<bound method Class.signature of Class('BaseCsvClient', 25, 58)>",
|
||||||
|
"docstring": "Abstract client responsible for retrieving csv bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full csv binary payload.\n - Raise retrieval-specific errors on failure.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.client.BaseCsvClient.fetch",
|
||||||
|
"signature": "<bound method Function.signature of Function('fetch', 40, 58)>",
|
||||||
|
"docstring": "Fetch raw csv bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"FileSystemCsvClient": {
|
||||||
|
"name": "FileSystemCsvClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.client.FileSystemCsvClient",
|
||||||
|
"signature": "<bound method Class.signature of Class('FileSystemCsvClient', 61, 97)>",
|
||||||
|
"docstring": "CSV client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads csv files directly from the disk and\n returns their raw binary contents.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.client.FileSystemCsvClient.fetch",
|
||||||
|
"signature": "<bound method Function.signature of Function('fetch', 72, 97)>",
|
||||||
|
"docstring": "Read a csv file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the csv file.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
576
docs/mcp/modules/omniread.csv.json
Normal file
576
docs/mcp/modules/omniread.csv.json
Normal file
@@ -0,0 +1,576 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.csv",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.csv",
|
||||||
|
"docstring": "# Summary\n\nCSV subpackage for OmniRead.\n\nProvides acquisition and parsing of comma-separated-value content:\n\n- `BaseCsvClient`: abstract backing-store client for csv bytes.\n- `FileSystemCsvClient`: local filesystem implementation.\n- `CsvScraper`: wraps fetched bytes into canonical `Content`.\n- `CsvParserBase`: content-type-enforcing parser contract.\n- `CsvParser`: generic string-row parser built on the standard csv module.",
|
||||||
|
"objects": {
|
||||||
|
"BaseCsvClient": {
|
||||||
|
"name": "BaseCsvClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.BaseCsvClient",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('BaseCsvClient', 'omniread.csv.client.BaseCsvClient')>",
|
||||||
|
"docstring": "Abstract client responsible for retrieving csv bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full csv binary payload.\n - Raise retrieval-specific errors on failure.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.BaseCsvClient.fetch",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.csv.client.BaseCsvClient.fetch')>",
|
||||||
|
"docstring": "Fetch raw csv bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"FileSystemCsvClient": {
|
||||||
|
"name": "FileSystemCsvClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.FileSystemCsvClient",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('FileSystemCsvClient', 'omniread.csv.client.FileSystemCsvClient')>",
|
||||||
|
"docstring": "CSV client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads csv files directly from the disk and\n returns their raw binary contents.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.FileSystemCsvClient.fetch",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.csv.client.FileSystemCsvClient.fetch')>",
|
||||||
|
"docstring": "Read a csv file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the csv file.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"CsvParser": {
|
||||||
|
"name": "CsvParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.CsvParser",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('CsvParser', 'omniread.csv.parser.CsvParser')>",
|
||||||
|
"docstring": "Generic csv parser producing string rows from the document.\n\nNotes:\n **Responsibilities:**\n\n - Decode the payload (UTF-8 with BOM support, Latin-1 fallback).\n - Detect the delimiter from a leading sample (`,` `;` tab `|`),\n defaulting to `,`.\n - Normalize cells into deterministic stripped string values.\n - Expose row extraction helpers mirroring `XlsxParser.rows`.\n\n **Constraints:**\n\n - All values are strings; consumers requiring typed values must\n convert on their side.\n - Quoted fields containing delimiters/newlines are handled by\n the standard ``csv`` module.",
|
||||||
|
"members": {
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.CsvParser.parse",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('parse', 'omniread.csv.parser.CsvParser.parse')>",
|
||||||
|
"docstring": "Parse the document into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the document."
|
||||||
|
},
|
||||||
|
"rows": {
|
||||||
|
"name": "rows",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.CsvParser.rows",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('rows', 'omniread.csv.parser.CsvParser.rows')>",
|
||||||
|
"docstring": "Extract normalized string rows from the document.\n\nArgs:\n skip_empty (bool):\n When True (default), rows whose cells are all blank are\n omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"CsvParserBase": {
|
||||||
|
"name": "CsvParserBase",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.CsvParserBase",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('CsvParserBase', 'omniread.csv.parser_base.CsvParserBase')>",
|
||||||
|
"docstring": "Base csv parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces csv content-type compatibility and provides\n the extension point for implementing concrete csv parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.CsvParserBase.supported_types",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('supported_types', 'omniread.csv.parser_base.CsvParserBase.supported_types')>",
|
||||||
|
"docstring": "Set of content types supported by this parser (CSV only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.CsvParserBase.parse",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('parse', 'omniread.csv.parser_base.CsvParserBase.parse')>",
|
||||||
|
"docstring": "Parse csv content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"CsvScraper": {
|
||||||
|
"name": "CsvScraper",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.CsvScraper",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('CsvScraper', 'omniread.csv.scraper.CsvScraper')>",
|
||||||
|
"docstring": "Scraper for csv documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw csv bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n CSV content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.CsvScraper.fetch",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.csv.scraper.CsvScraper.fetch')>",
|
||||||
|
"docstring": "Fetch a csv document from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw csv bytes, source\n identifier, CSV content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"client": {
|
||||||
|
"name": "client",
|
||||||
|
"kind": "module",
|
||||||
|
"path": "omniread.csv.client",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "# Summary\n\nCSV client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw\ncomma-separated-value document bytes from a concrete backing store.\n\nClients provide low-level access to csv binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems",
|
||||||
|
"members": {
|
||||||
|
"ABC": {
|
||||||
|
"name": "ABC",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.csv.client.ABC",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('ABC', 'abc.ABC')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"abstractmethod": {
|
||||||
|
"name": "abstractmethod",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.csv.client.abstractmethod",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Path": {
|
||||||
|
"name": "Path",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.csv.client.Path",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Path', 'pathlib.Path')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Any": {
|
||||||
|
"name": "Any",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.csv.client.Any",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"BaseCsvClient": {
|
||||||
|
"name": "BaseCsvClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.client.BaseCsvClient",
|
||||||
|
"signature": "<bound method Class.signature of Class('BaseCsvClient', 25, 58)>",
|
||||||
|
"docstring": "Abstract client responsible for retrieving csv bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full csv binary payload.\n - Raise retrieval-specific errors on failure.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.client.BaseCsvClient.fetch",
|
||||||
|
"signature": "<bound method Function.signature of Function('fetch', 40, 58)>",
|
||||||
|
"docstring": "Fetch raw csv bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"FileSystemCsvClient": {
|
||||||
|
"name": "FileSystemCsvClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.client.FileSystemCsvClient",
|
||||||
|
"signature": "<bound method Class.signature of Class('FileSystemCsvClient', 61, 97)>",
|
||||||
|
"docstring": "CSV client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads csv files directly from the disk and\n returns their raw binary contents.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.client.FileSystemCsvClient.fetch",
|
||||||
|
"signature": "<bound method Function.signature of Function('fetch', 72, 97)>",
|
||||||
|
"docstring": "Read a csv file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the csv file.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"parser": {
|
||||||
|
"name": "parser",
|
||||||
|
"kind": "module",
|
||||||
|
"path": "omniread.csv.parser",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "# Summary\n\nCSV parser implementations for OmniRead.\n\nThis module provides a concrete, generic parser for comma-separated-value\ndocuments. It exposes records as lists of string cells so downstream\nconsumers can interpret tabular content without depending on the ``csv``\nmodule directly.\n\nThe parser is intentionally statement-agnostic: it performs no header\ndetection or column interpretation beyond basic cell normalization and\ndelimiter detection.",
|
||||||
|
"members": {
|
||||||
|
"Sniffer": {
|
||||||
|
"name": "Sniffer",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.csv.parser.Sniffer",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Sniffer', 'csv.Sniffer')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"reader": {
|
||||||
|
"name": "reader",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.csv.parser.reader",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('reader', 'csv.reader')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"StringIO": {
|
||||||
|
"name": "StringIO",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.csv.parser.StringIO",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('StringIO', 'io.StringIO')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.parser.Content",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Content', 'omniread.core.content.Content')>",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser.Content.raw",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('raw', 'omniread.core.content.Content.raw')>",
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser.Content.source",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('source', 'omniread.core.content.Content.source')>",
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser.Content.content_type",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('content_type', 'omniread.core.content.Content.content_type')>",
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser.Content.metadata",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('metadata', 'omniread.core.content.Content.metadata')>",
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"CsvParserBase": {
|
||||||
|
"name": "CsvParserBase",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.parser.CsvParserBase",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('CsvParserBase', 'omniread.csv.parser_base.CsvParserBase')>",
|
||||||
|
"docstring": "Base csv parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces csv content-type compatibility and provides\n the extension point for implementing concrete csv parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser.CsvParserBase.supported_types",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('supported_types', 'omniread.csv.parser_base.CsvParserBase.supported_types')>",
|
||||||
|
"docstring": "Set of content types supported by this parser (CSV only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.parser.CsvParserBase.parse",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('parse', 'omniread.csv.parser_base.CsvParserBase.parse')>",
|
||||||
|
"docstring": "Parse csv content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"CsvParser": {
|
||||||
|
"name": "CsvParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.parser.CsvParser",
|
||||||
|
"signature": "<bound method Class.signature of Class('CsvParser', 24, 102)>",
|
||||||
|
"docstring": "Generic csv parser producing string rows from the document.\n\nNotes:\n **Responsibilities:**\n\n - Decode the payload (UTF-8 with BOM support, Latin-1 fallback).\n - Detect the delimiter from a leading sample (`,` `;` tab `|`),\n defaulting to `,`.\n - Normalize cells into deterministic stripped string values.\n - Expose row extraction helpers mirroring `XlsxParser.rows`.\n\n **Constraints:**\n\n - All values are strings; consumers requiring typed values must\n convert on their side.\n - Quoted fields containing delimiters/newlines are handled by\n the standard ``csv`` module.",
|
||||||
|
"members": {
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.parser.CsvParser.parse",
|
||||||
|
"signature": "<bound method Function.signature of Function('parse', 57, 65)>",
|
||||||
|
"docstring": "Parse the document into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the document."
|
||||||
|
},
|
||||||
|
"rows": {
|
||||||
|
"name": "rows",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.parser.CsvParser.rows",
|
||||||
|
"signature": "<bound method Function.signature of Function('rows', 67, 95)>",
|
||||||
|
"docstring": "Extract normalized string rows from the document.\n\nArgs:\n skip_empty (bool):\n When True (default), rows whose cells are all blank are\n omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"parser_base": {
|
||||||
|
"name": "parser_base",
|
||||||
|
"kind": "module",
|
||||||
|
"path": "omniread.csv.parser_base",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "# Summary\n\nCSV parser base implementation for OmniRead.\n\nThis module defines the **CSV-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for\ncomma-separated-value documents.",
|
||||||
|
"members": {
|
||||||
|
"abstractmethod": {
|
||||||
|
"name": "abstractmethod",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.csv.parser_base.abstractmethod",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Generic": {
|
||||||
|
"name": "Generic",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.csv.parser_base.Generic",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Generic', 'typing.Generic')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"TypeVar": {
|
||||||
|
"name": "TypeVar",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.csv.parser_base.TypeVar",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('TypeVar', 'typing.TypeVar')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('ContentType', 'omniread.core.content.ContentType')>",
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType.HTML",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('HTML', 'omniread.core.content.ContentType.HTML')>",
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType.PDF",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType.XLSX",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XLSX', 'omniread.core.content.ContentType.XLSX')>",
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType.CSV",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('CSV', 'omniread.core.content.ContentType.CSV')>",
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType.JSON",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('JSON', 'omniread.core.content.ContentType.JSON')>",
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType.XML",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XML', 'omniread.core.content.ContentType.XML')>",
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseParser": {
|
||||||
|
"name": "BaseParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.parser_base.BaseParser",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('BaseParser', 'omniread.core.parser.BaseParser')>",
|
||||||
|
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.BaseParser.supported_types",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('supported_types', 'omniread.core.parser.BaseParser.supported_types')>",
|
||||||
|
"docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic."
|
||||||
|
},
|
||||||
|
"content": {
|
||||||
|
"name": "content",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.BaseParser.content",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('content', 'omniread.core.parser.BaseParser.content')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.parser_base.BaseParser.parse",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('parse', 'omniread.core.parser.BaseParser.parse')>",
|
||||||
|
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
|
||||||
|
},
|
||||||
|
"supports": {
|
||||||
|
"name": "supports",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.parser_base.BaseParser.supports",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('supports', 'omniread.core.parser.BaseParser.supports')>",
|
||||||
|
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"T": {
|
||||||
|
"name": "T",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.T",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"CsvParserBase": {
|
||||||
|
"name": "CsvParserBase",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.parser_base.CsvParserBase",
|
||||||
|
"signature": "<bound method Class.signature of Class('CsvParserBase', 20, 55)>",
|
||||||
|
"docstring": "Base csv parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces csv content-type compatibility and provides\n the extension point for implementing concrete csv parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.CsvParserBase.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser (CSV only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.parser_base.CsvParserBase.parse",
|
||||||
|
"signature": "<bound method Function.signature of Function('parse', 42, 55)>",
|
||||||
|
"docstring": "Parse csv content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"scraper": {
|
||||||
|
"name": "scraper",
|
||||||
|
"kind": "module",
|
||||||
|
"path": "omniread.csv.scraper",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "# Summary\n\nCSV scraper for OmniRead.\n\nThis module defines the scraper responsible for acquiring raw\ncomma-separated-value document content from a backing store via a\nconfigured client.\n\nThe scraper does not interpret or parse the acquired bytes; it wraps them in\nthe canonical `Content` model.",
|
||||||
|
"members": {
|
||||||
|
"Mapping": {
|
||||||
|
"name": "Mapping",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.csv.scraper.Mapping",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Mapping', 'collections.abc.Mapping')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Any": {
|
||||||
|
"name": "Any",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.csv.scraper.Any",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.scraper.Content",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Content', 'omniread.core.content.Content')>",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.Content.raw",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('raw', 'omniread.core.content.Content.raw')>",
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.Content.source",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('source', 'omniread.core.content.Content.source')>",
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.Content.content_type",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('content_type', 'omniread.core.content.Content.content_type')>",
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.Content.metadata",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('metadata', 'omniread.core.content.Content.metadata')>",
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.scraper.ContentType",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('ContentType', 'omniread.core.content.ContentType')>",
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.ContentType.HTML",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('HTML', 'omniread.core.content.ContentType.HTML')>",
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.ContentType.PDF",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.ContentType.XLSX",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XLSX', 'omniread.core.content.ContentType.XLSX')>",
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.ContentType.CSV",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('CSV', 'omniread.core.content.ContentType.CSV')>",
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.ContentType.JSON",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('JSON', 'omniread.core.content.ContentType.JSON')>",
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.ContentType.XML",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XML', 'omniread.core.content.ContentType.XML')>",
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseCsvClient": {
|
||||||
|
"name": "BaseCsvClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.scraper.BaseCsvClient",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('BaseCsvClient', 'omniread.csv.client.BaseCsvClient')>",
|
||||||
|
"docstring": "Abstract client responsible for retrieving csv bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full csv binary payload.\n - Raise retrieval-specific errors on failure.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.scraper.BaseCsvClient.fetch",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.csv.client.BaseCsvClient.fetch')>",
|
||||||
|
"docstring": "Fetch raw csv bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"CsvScraper": {
|
||||||
|
"name": "CsvScraper",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.scraper.CsvScraper",
|
||||||
|
"signature": "<bound method Class.signature of Class('CsvScraper', 22, 81)>",
|
||||||
|
"docstring": "Scraper for csv documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw csv bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n CSV content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.scraper.CsvScraper.fetch",
|
||||||
|
"signature": "<bound method Function.signature of Function('fetch', 49, 81)>",
|
||||||
|
"docstring": "Fetch a csv document from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw csv bytes, source\n identifier, CSV content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
113
docs/mcp/modules/omniread.csv.parser.json
Normal file
113
docs/mcp/modules/omniread.csv.parser.json
Normal file
@@ -0,0 +1,113 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.csv.parser",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.csv.parser",
|
||||||
|
"docstring": "# Summary\n\nCSV parser implementations for OmniRead.\n\nThis module provides a concrete, generic parser for comma-separated-value\ndocuments. It exposes records as lists of string cells so downstream\nconsumers can interpret tabular content without depending on the ``csv``\nmodule directly.\n\nThe parser is intentionally statement-agnostic: it performs no header\ndetection or column interpretation beyond basic cell normalization and\ndelimiter detection.",
|
||||||
|
"objects": {
|
||||||
|
"Sniffer": {
|
||||||
|
"name": "Sniffer",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.csv.parser.Sniffer",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Sniffer', 'csv.Sniffer')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"reader": {
|
||||||
|
"name": "reader",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.csv.parser.reader",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('reader', 'csv.reader')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"StringIO": {
|
||||||
|
"name": "StringIO",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.csv.parser.StringIO",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('StringIO', 'io.StringIO')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.parser.Content",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Content', 'omniread.core.content.Content')>",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser.Content.raw",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('raw', 'omniread.core.content.Content.raw')>",
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser.Content.source",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('source', 'omniread.core.content.Content.source')>",
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser.Content.content_type",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('content_type', 'omniread.core.content.Content.content_type')>",
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser.Content.metadata",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('metadata', 'omniread.core.content.Content.metadata')>",
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"CsvParserBase": {
|
||||||
|
"name": "CsvParserBase",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.parser.CsvParserBase",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('CsvParserBase', 'omniread.csv.parser_base.CsvParserBase')>",
|
||||||
|
"docstring": "Base csv parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces csv content-type compatibility and provides\n the extension point for implementing concrete csv parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser.CsvParserBase.supported_types",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('supported_types', 'omniread.csv.parser_base.CsvParserBase.supported_types')>",
|
||||||
|
"docstring": "Set of content types supported by this parser (CSV only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.parser.CsvParserBase.parse",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('parse', 'omniread.csv.parser_base.CsvParserBase.parse')>",
|
||||||
|
"docstring": "Parse csv content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"CsvParser": {
|
||||||
|
"name": "CsvParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.parser.CsvParser",
|
||||||
|
"signature": "<bound method Class.signature of Class('CsvParser', 24, 102)>",
|
||||||
|
"docstring": "Generic csv parser producing string rows from the document.\n\nNotes:\n **Responsibilities:**\n\n - Decode the payload (UTF-8 with BOM support, Latin-1 fallback).\n - Detect the delimiter from a leading sample (`,` `;` tab `|`),\n defaulting to `,`.\n - Normalize cells into deterministic stripped string values.\n - Expose row extraction helpers mirroring `XlsxParser.rows`.\n\n **Constraints:**\n\n - All values are strings; consumers requiring typed values must\n convert on their side.\n - Quoted fields containing delimiters/newlines are handled by\n the standard ``csv`` module.",
|
||||||
|
"members": {
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.parser.CsvParser.parse",
|
||||||
|
"signature": "<bound method Function.signature of Function('parse', 57, 65)>",
|
||||||
|
"docstring": "Parse the document into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the document."
|
||||||
|
},
|
||||||
|
"rows": {
|
||||||
|
"name": "rows",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.parser.CsvParser.rows",
|
||||||
|
"signature": "<bound method Function.signature of Function('rows', 67, 95)>",
|
||||||
|
"docstring": "Extract normalized string rows from the document.\n\nArgs:\n skip_empty (bool):\n When True (default), rows whose cells are all blank are\n omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
148
docs/mcp/modules/omniread.csv.parser_base.json
Normal file
148
docs/mcp/modules/omniread.csv.parser_base.json
Normal file
@@ -0,0 +1,148 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.csv.parser_base",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.csv.parser_base",
|
||||||
|
"docstring": "# Summary\n\nCSV parser base implementation for OmniRead.\n\nThis module defines the **CSV-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for\ncomma-separated-value documents.",
|
||||||
|
"objects": {
|
||||||
|
"abstractmethod": {
|
||||||
|
"name": "abstractmethod",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.csv.parser_base.abstractmethod",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Generic": {
|
||||||
|
"name": "Generic",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.csv.parser_base.Generic",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Generic', 'typing.Generic')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"TypeVar": {
|
||||||
|
"name": "TypeVar",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.csv.parser_base.TypeVar",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('TypeVar', 'typing.TypeVar')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('ContentType', 'omniread.core.content.ContentType')>",
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType.HTML",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('HTML', 'omniread.core.content.ContentType.HTML')>",
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType.PDF",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType.XLSX",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XLSX', 'omniread.core.content.ContentType.XLSX')>",
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType.CSV",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('CSV', 'omniread.core.content.ContentType.CSV')>",
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType.JSON",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('JSON', 'omniread.core.content.ContentType.JSON')>",
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.ContentType.XML",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XML', 'omniread.core.content.ContentType.XML')>",
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseParser": {
|
||||||
|
"name": "BaseParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.parser_base.BaseParser",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('BaseParser', 'omniread.core.parser.BaseParser')>",
|
||||||
|
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.BaseParser.supported_types",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('supported_types', 'omniread.core.parser.BaseParser.supported_types')>",
|
||||||
|
"docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic."
|
||||||
|
},
|
||||||
|
"content": {
|
||||||
|
"name": "content",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.BaseParser.content",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('content', 'omniread.core.parser.BaseParser.content')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.parser_base.BaseParser.parse",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('parse', 'omniread.core.parser.BaseParser.parse')>",
|
||||||
|
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
|
||||||
|
},
|
||||||
|
"supports": {
|
||||||
|
"name": "supports",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.parser_base.BaseParser.supports",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('supports', 'omniread.core.parser.BaseParser.supports')>",
|
||||||
|
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"T": {
|
||||||
|
"name": "T",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.T",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"CsvParserBase": {
|
||||||
|
"name": "CsvParserBase",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.parser_base.CsvParserBase",
|
||||||
|
"signature": "<bound method Class.signature of Class('CsvParserBase', 20, 55)>",
|
||||||
|
"docstring": "Base csv parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces csv content-type compatibility and provides\n the extension point for implementing concrete csv parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.parser_base.CsvParserBase.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser (CSV only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.parser_base.CsvParserBase.parse",
|
||||||
|
"signature": "<bound method Function.signature of Function('parse', 42, 55)>",
|
||||||
|
"docstring": "Parse csv content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
143
docs/mcp/modules/omniread.csv.scraper.json
Normal file
143
docs/mcp/modules/omniread.csv.scraper.json
Normal file
@@ -0,0 +1,143 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.csv.scraper",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.csv.scraper",
|
||||||
|
"docstring": "# Summary\n\nCSV scraper for OmniRead.\n\nThis module defines the scraper responsible for acquiring raw\ncomma-separated-value document content from a backing store via a\nconfigured client.\n\nThe scraper does not interpret or parse the acquired bytes; it wraps them in\nthe canonical `Content` model.",
|
||||||
|
"objects": {
|
||||||
|
"Mapping": {
|
||||||
|
"name": "Mapping",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.csv.scraper.Mapping",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Mapping', 'collections.abc.Mapping')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Any": {
|
||||||
|
"name": "Any",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.csv.scraper.Any",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.scraper.Content",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Content', 'omniread.core.content.Content')>",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.Content.raw",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('raw', 'omniread.core.content.Content.raw')>",
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.Content.source",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('source', 'omniread.core.content.Content.source')>",
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.Content.content_type",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('content_type', 'omniread.core.content.Content.content_type')>",
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.Content.metadata",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('metadata', 'omniread.core.content.Content.metadata')>",
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.scraper.ContentType",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('ContentType', 'omniread.core.content.ContentType')>",
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.ContentType.HTML",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('HTML', 'omniread.core.content.ContentType.HTML')>",
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.ContentType.PDF",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.ContentType.XLSX",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XLSX', 'omniread.core.content.ContentType.XLSX')>",
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.ContentType.CSV",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('CSV', 'omniread.core.content.ContentType.CSV')>",
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.ContentType.JSON",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('JSON', 'omniread.core.content.ContentType.JSON')>",
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.csv.scraper.ContentType.XML",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XML', 'omniread.core.content.ContentType.XML')>",
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseCsvClient": {
|
||||||
|
"name": "BaseCsvClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.scraper.BaseCsvClient",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('BaseCsvClient', 'omniread.csv.client.BaseCsvClient')>",
|
||||||
|
"docstring": "Abstract client responsible for retrieving csv bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full csv binary payload.\n - Raise retrieval-specific errors on failure.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.scraper.BaseCsvClient.fetch",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.csv.client.BaseCsvClient.fetch')>",
|
||||||
|
"docstring": "Fetch raw csv bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"CsvScraper": {
|
||||||
|
"name": "CsvScraper",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.csv.scraper.CsvScraper",
|
||||||
|
"signature": "<bound method Class.signature of Class('CsvScraper', 22, 81)>",
|
||||||
|
"docstring": "Scraper for csv documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw csv bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n CSV content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.csv.scraper.CsvScraper.fetch",
|
||||||
|
"signature": "<bound method Function.signature of Function('fetch', 49, 81)>",
|
||||||
|
"docstring": "Fetch a csv document from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw csv bytes, source\n identifier, CSV content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -30,7 +30,7 @@
|
|||||||
"kind": "function",
|
"kind": "function",
|
||||||
"path": "omniread.html.HTMLScraper.fetch",
|
"path": "omniread.html.HTMLScraper.fetch",
|
||||||
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.html.scraper.HTMLScraper.fetch')>",
|
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.html.scraper.HTMLScraper.fetch')>",
|
||||||
"docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML."
|
"docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML."
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -67,7 +67,7 @@
|
|||||||
"kind": "function",
|
"kind": "function",
|
||||||
"path": "omniread.html.HTMLParser.parse_link",
|
"path": "omniread.html.HTMLParser.parse_link",
|
||||||
"signature": "<bound method Alias.signature of Alias('parse_link', 'omniread.html.parser.HTMLParser.parse_link')>",
|
"signature": "<bound method Alias.signature of Alias('parse_link', 'omniread.html.parser.HTMLParser.parse_link')>",
|
||||||
"docstring": "Extract the hyperlink reference from an `<a>` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n Optional[str]:\n The value of the `href` attribute, or None if absent."
|
"docstring": "Extract the hyperlink reference from an `<a>` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n str | None:\n The value of the `href` attribute, or None if absent."
|
||||||
},
|
},
|
||||||
"parse_table": {
|
"parse_table": {
|
||||||
"name": "parse_table",
|
"name": "parse_table",
|
||||||
@@ -92,6 +92,13 @@
|
|||||||
"signature": null,
|
"signature": null,
|
||||||
"docstring": "# Summary\n\nHTML parser base implementations for OmniRead.\n\nThis module provides reusable HTML parsing utilities built on top of\nthe abstract parser contracts defined in `omniread.core.parser`.\n\nIt supplies:\n\n- Content-type enforcement for HTML inputs\n- BeautifulSoup initialization and lifecycle management\n- Common helper methods for extracting structured data from HTML elements\n\nConcrete parsers must subclass `HTMLParser` and implement the `parse()` method\nto return a structured representation appropriate for their use case.",
|
"docstring": "# Summary\n\nHTML parser base implementations for OmniRead.\n\nThis module provides reusable HTML parsing utilities built on top of\nthe abstract parser contracts defined in `omniread.core.parser`.\n\nIt supplies:\n\n- Content-type enforcement for HTML inputs\n- BeautifulSoup initialization and lifecycle management\n- Common helper methods for extracting structured data from HTML elements\n\nConcrete parsers must subclass `HTMLParser` and implement the `parse()` method\nto return a structured representation appropriate for their use case.",
|
||||||
"members": {
|
"members": {
|
||||||
|
"abstractmethod": {
|
||||||
|
"name": "abstractmethod",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.html.parser.abstractmethod",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
"Any": {
|
"Any": {
|
||||||
"name": "Any",
|
"name": "Any",
|
||||||
"kind": "alias",
|
"kind": "alias",
|
||||||
@@ -113,20 +120,6 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('TypeVar', 'typing.TypeVar')>",
|
"signature": "<bound method Alias.signature of Alias('TypeVar', 'typing.TypeVar')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
"Optional": {
|
|
||||||
"name": "Optional",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.html.parser.Optional",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('Optional', 'typing.Optional')>",
|
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"abstractmethod": {
|
|
||||||
"name": "abstractmethod",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.html.parser.abstractmethod",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
|
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"BeautifulSoup": {
|
"BeautifulSoup": {
|
||||||
"name": "BeautifulSoup",
|
"name": "BeautifulSoup",
|
||||||
"kind": "alias",
|
"kind": "alias",
|
||||||
@@ -141,43 +134,6 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('Tag', 'bs4.Tag')>",
|
"signature": "<bound method Alias.signature of Alias('Tag', 'bs4.Tag')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
"ContentType": {
|
|
||||||
"name": "ContentType",
|
|
||||||
"kind": "class",
|
|
||||||
"path": "omniread.html.parser.ContentType",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('ContentType', 'omniread.core.content.ContentType')>",
|
|
||||||
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
|
||||||
"members": {
|
|
||||||
"HTML": {
|
|
||||||
"name": "HTML",
|
|
||||||
"kind": "attribute",
|
|
||||||
"path": "omniread.html.parser.ContentType.HTML",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('HTML', 'omniread.core.content.ContentType.HTML')>",
|
|
||||||
"docstring": "HTML document content."
|
|
||||||
},
|
|
||||||
"PDF": {
|
|
||||||
"name": "PDF",
|
|
||||||
"kind": "attribute",
|
|
||||||
"path": "omniread.html.parser.ContentType.PDF",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
|
||||||
"docstring": "PDF document content."
|
|
||||||
},
|
|
||||||
"JSON": {
|
|
||||||
"name": "JSON",
|
|
||||||
"kind": "attribute",
|
|
||||||
"path": "omniread.html.parser.ContentType.JSON",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('JSON', 'omniread.core.content.ContentType.JSON')>",
|
|
||||||
"docstring": "JSON document content."
|
|
||||||
},
|
|
||||||
"XML": {
|
|
||||||
"name": "XML",
|
|
||||||
"kind": "attribute",
|
|
||||||
"path": "omniread.html.parser.ContentType.XML",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('XML', 'omniread.core.content.ContentType.XML')>",
|
|
||||||
"docstring": "XML document content."
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"Content": {
|
"Content": {
|
||||||
"name": "Content",
|
"name": "Content",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
@@ -215,6 +171,57 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.html.parser.ContentType",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('ContentType', 'omniread.core.content.ContentType')>",
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.ContentType.HTML",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('HTML', 'omniread.core.content.ContentType.HTML')>",
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.ContentType.PDF",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.ContentType.XLSX",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XLSX', 'omniread.core.content.ContentType.XLSX')>",
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.ContentType.CSV",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('CSV', 'omniread.core.content.ContentType.CSV')>",
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.ContentType.JSON",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('JSON', 'omniread.core.content.ContentType.JSON')>",
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.ContentType.XML",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XML', 'omniread.core.content.ContentType.XML')>",
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"BaseParser": {
|
"BaseParser": {
|
||||||
"name": "BaseParser",
|
"name": "BaseParser",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
@@ -263,7 +270,7 @@
|
|||||||
"name": "HTMLParser",
|
"name": "HTMLParser",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
"path": "omniread.html.parser.HTMLParser",
|
"path": "omniread.html.parser.HTMLParser",
|
||||||
"signature": "<bound method Class.signature of Class('HTMLParser', 30, 205)>",
|
"signature": "<bound method Class.signature of Class('HTMLParser', 30, 202)>",
|
||||||
"docstring": "Base HTML parser.\n\nNotes:\n **Responsibilities:**\n\n - This class extends the core `BaseParser` with HTML-specific behavior,\n including DOM parsing via BeautifulSoup and reusable extraction helpers.\n - Provides reusable helpers for HTML extraction. Concrete parsers must\n explicitly define the return type.\n\n **Guarantees:**\n\n - Accepts only HTML content.\n - Owns a parsed BeautifulSoup DOM tree.\n - Provides pure helper utilities for common HTML structures.\n\n **Constraints:**\n\n - Concrete subclasses must define the output type `T` and implement\n the `parse()` method.",
|
"docstring": "Base HTML parser.\n\nNotes:\n **Responsibilities:**\n\n - This class extends the core `BaseParser` with HTML-specific behavior,\n including DOM parsing via BeautifulSoup and reusable extraction helpers.\n - Provides reusable helpers for HTML extraction. Concrete parsers must\n explicitly define the return type.\n\n **Guarantees:**\n\n - Accepts only HTML content.\n - Owns a parsed BeautifulSoup DOM tree.\n - Provides pure helper utilities for common HTML structures.\n\n **Constraints:**\n\n - Concrete subclasses must define the output type `T` and implement\n the `parse()` method.",
|
||||||
"members": {
|
"members": {
|
||||||
"supported_types": {
|
"supported_types": {
|
||||||
@@ -292,37 +299,23 @@
|
|||||||
"kind": "function",
|
"kind": "function",
|
||||||
"path": "omniread.html.parser.HTMLParser.parse_link",
|
"path": "omniread.html.parser.HTMLParser.parse_link",
|
||||||
"signature": "<bound method Function.signature of Function('parse_link', 119, 132)>",
|
"signature": "<bound method Function.signature of Function('parse_link', 119, 132)>",
|
||||||
"docstring": "Extract the hyperlink reference from an `<a>` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n Optional[str]:\n The value of the `href` attribute, or None if absent."
|
"docstring": "Extract the hyperlink reference from an `<a>` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n str | None:\n The value of the `href` attribute, or None if absent."
|
||||||
},
|
},
|
||||||
"parse_table": {
|
"parse_table": {
|
||||||
"name": "parse_table",
|
"name": "parse_table",
|
||||||
"kind": "function",
|
"kind": "function",
|
||||||
"path": "omniread.html.parser.HTMLParser.parse_table",
|
"path": "omniread.html.parser.HTMLParser.parse_table",
|
||||||
"signature": "<bound method Function.signature of Function('parse_table', 134, 155)>",
|
"signature": "<bound method Function.signature of Function('parse_table', 134, 152)>",
|
||||||
"docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table (Tag):\n BeautifulSoup tag representing a `<table>`.\n\nReturns:\n list[list[str]]:\n A list of rows, where each row is a list of cell text values."
|
"docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table (Tag):\n BeautifulSoup tag representing a `<table>`.\n\nReturns:\n list[list[str]]:\n A list of rows, where each row is a list of cell text values."
|
||||||
},
|
},
|
||||||
"parse_meta": {
|
"parse_meta": {
|
||||||
"name": "parse_meta",
|
"name": "parse_meta",
|
||||||
"kind": "function",
|
"kind": "function",
|
||||||
"path": "omniread.html.parser.HTMLParser.parse_meta",
|
"path": "omniread.html.parser.HTMLParser.parse_meta",
|
||||||
"signature": "<bound method Function.signature of Function('parse_meta', 177, 205)>",
|
"signature": "<bound method Function.signature of Function('parse_meta', 174, 202)>",
|
||||||
"docstring": "Extract high-level metadata from the HTML document.\n\nReturns:\n dict[str, Any]:\n Dictionary containing extracted metadata.\n\nNotes:\n **Responsibilities:**\n\n - Extract high-level metadata from the HTML document.\n - This includes: Document title, `<meta>` tag name/property to\n content mappings."
|
"docstring": "Extract high-level metadata from the HTML document.\n\nReturns:\n dict[str, Any]:\n Dictionary containing extracted metadata.\n\nNotes:\n **Responsibilities:**\n\n - Extract high-level metadata from the HTML document.\n - This includes: Document title, `<meta>` tag name/property to\n content mappings."
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
|
||||||
"list": {
|
|
||||||
"name": "list",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.html.parser.list",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('list', 'typing.list')>",
|
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"dict": {
|
|
||||||
"name": "dict",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.html.parser.dict",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('dict', 'typing.dict')>",
|
|
||||||
"docstring": null
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -333,11 +326,11 @@
|
|||||||
"signature": null,
|
"signature": null,
|
||||||
"docstring": "# Summary\n\nHTML scraping implementation for OmniRead.\n\nThis module provides an HTTP-based scraper for retrieving HTML documents.\nIt implements the core `BaseScraper` contract using `httpx` as the transport\nlayer.\n\nThis scraper is responsible for:\n\n- Fetching raw HTML bytes over HTTP(S)\n- Validating response content type\n- Attaching HTTP metadata to the returned content\n\nThis scraper is not responsible for:\n\n- Parsing or interpreting HTML\n- Retrying failed requests\n- Managing crawl policies or rate limiting",
|
"docstring": "# Summary\n\nHTML scraping implementation for OmniRead.\n\nThis module provides an HTTP-based scraper for retrieving HTML documents.\nIt implements the core `BaseScraper` contract using `httpx` as the transport\nlayer.\n\nThis scraper is responsible for:\n\n- Fetching raw HTML bytes over HTTP(S)\n- Validating response content type\n- Attaching HTTP metadata to the returned content\n\nThis scraper is not responsible for:\n\n- Parsing or interpreting HTML\n- Retrying failed requests\n- Managing crawl policies or rate limiting",
|
||||||
"members": {
|
"members": {
|
||||||
"httpx": {
|
"Mapping": {
|
||||||
"name": "httpx",
|
"name": "Mapping",
|
||||||
"kind": "alias",
|
"kind": "alias",
|
||||||
"path": "omniread.html.scraper.httpx",
|
"path": "omniread.html.scraper.Mapping",
|
||||||
"signature": "<bound method Alias.signature of Alias('httpx', 'httpx')>",
|
"signature": "<bound method Alias.signature of Alias('Mapping', 'collections.abc.Mapping')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
"Any": {
|
"Any": {
|
||||||
@@ -347,18 +340,11 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
"Mapping": {
|
"httpx": {
|
||||||
"name": "Mapping",
|
"name": "httpx",
|
||||||
"kind": "alias",
|
"kind": "alias",
|
||||||
"path": "omniread.html.scraper.Mapping",
|
"path": "omniread.html.scraper.httpx",
|
||||||
"signature": "<bound method Alias.signature of Alias('Mapping', 'typing.Mapping')>",
|
"signature": "<bound method Alias.signature of Alias('httpx', 'httpx')>",
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"Optional": {
|
|
||||||
"name": "Optional",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.html.scraper.Optional",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('Optional', 'typing.Optional')>",
|
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
"Content": {
|
"Content": {
|
||||||
@@ -419,6 +405,20 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
||||||
"docstring": "PDF document content."
|
"docstring": "PDF document content."
|
||||||
},
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.ContentType.XLSX",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XLSX', 'omniread.core.content.ContentType.XLSX')>",
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.ContentType.CSV",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('CSV', 'omniread.core.content.ContentType.CSV')>",
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
"JSON": {
|
"JSON": {
|
||||||
"name": "JSON",
|
"name": "JSON",
|
||||||
"kind": "attribute",
|
"kind": "attribute",
|
||||||
@@ -447,7 +447,7 @@
|
|||||||
"kind": "function",
|
"kind": "function",
|
||||||
"path": "omniread.html.scraper.BaseScraper.fetch",
|
"path": "omniread.html.scraper.BaseScraper.fetch",
|
||||||
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.core.scraper.BaseScraper.fetch')>",
|
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.core.scraper.BaseScraper.fetch')>",
|
||||||
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
|
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -455,7 +455,7 @@
|
|||||||
"name": "HTMLScraper",
|
"name": "HTMLScraper",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
"path": "omniread.html.scraper.HTMLScraper",
|
"path": "omniread.html.scraper.HTMLScraper",
|
||||||
"signature": "<bound method Class.signature of Class('HTMLScraper', 30, 143)>",
|
"signature": "<bound method Class.signature of Class('HTMLScraper', 32, 143)>",
|
||||||
"docstring": "Base HTML scraper using `httpx`.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns\n them as raw content wrapped in a `Content` object.\n - Fetches raw bytes and metadata only.\n - The scraper uses `httpx.Client` for HTTP requests, enforces an\n HTML content type, and preserves HTTP response metadata.\n\n **Constraints:**\n\n - The scraper does not: Parse HTML, perform retries or backoff,\n handle non-HTML responses.",
|
"docstring": "Base HTML scraper using `httpx`.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns\n them as raw content wrapped in a `Content` object.\n - Fetches raw bytes and metadata only.\n - The scraper uses `httpx.Client` for HTTP requests, enforces an\n HTML content type, and preserves HTTP response metadata.\n\n **Constraints:**\n\n - The scraper does not: Parse HTML, perform retries or backoff,\n handle non-HTML responses.",
|
||||||
"members": {
|
"members": {
|
||||||
"content_type": {
|
"content_type": {
|
||||||
@@ -469,7 +469,7 @@
|
|||||||
"name": "validate_content_type",
|
"name": "validate_content_type",
|
||||||
"kind": "function",
|
"kind": "function",
|
||||||
"path": "omniread.html.scraper.HTMLScraper.validate_content_type",
|
"path": "omniread.html.scraper.HTMLScraper.validate_content_type",
|
||||||
"signature": "<bound method Function.signature of Function('validate_content_type', 78, 102)>",
|
"signature": "<bound method Function.signature of Function('validate_content_type', 80, 102)>",
|
||||||
"docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content."
|
"docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content."
|
||||||
},
|
},
|
||||||
"fetch": {
|
"fetch": {
|
||||||
@@ -477,7 +477,7 @@
|
|||||||
"kind": "function",
|
"kind": "function",
|
||||||
"path": "omniread.html.scraper.HTMLScraper.fetch",
|
"path": "omniread.html.scraper.HTMLScraper.fetch",
|
||||||
"signature": "<bound method Function.signature of Function('fetch', 104, 143)>",
|
"signature": "<bound method Function.signature of Function('fetch', 104, 143)>",
|
||||||
"docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML."
|
"docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML."
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -4,6 +4,13 @@
|
|||||||
"path": "omniread.html.parser",
|
"path": "omniread.html.parser",
|
||||||
"docstring": "# Summary\n\nHTML parser base implementations for OmniRead.\n\nThis module provides reusable HTML parsing utilities built on top of\nthe abstract parser contracts defined in `omniread.core.parser`.\n\nIt supplies:\n\n- Content-type enforcement for HTML inputs\n- BeautifulSoup initialization and lifecycle management\n- Common helper methods for extracting structured data from HTML elements\n\nConcrete parsers must subclass `HTMLParser` and implement the `parse()` method\nto return a structured representation appropriate for their use case.",
|
"docstring": "# Summary\n\nHTML parser base implementations for OmniRead.\n\nThis module provides reusable HTML parsing utilities built on top of\nthe abstract parser contracts defined in `omniread.core.parser`.\n\nIt supplies:\n\n- Content-type enforcement for HTML inputs\n- BeautifulSoup initialization and lifecycle management\n- Common helper methods for extracting structured data from HTML elements\n\nConcrete parsers must subclass `HTMLParser` and implement the `parse()` method\nto return a structured representation appropriate for their use case.",
|
||||||
"objects": {
|
"objects": {
|
||||||
|
"abstractmethod": {
|
||||||
|
"name": "abstractmethod",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.html.parser.abstractmethod",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
"Any": {
|
"Any": {
|
||||||
"name": "Any",
|
"name": "Any",
|
||||||
"kind": "alias",
|
"kind": "alias",
|
||||||
@@ -25,20 +32,6 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('TypeVar', 'typing.TypeVar')>",
|
"signature": "<bound method Alias.signature of Alias('TypeVar', 'typing.TypeVar')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
"Optional": {
|
|
||||||
"name": "Optional",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.html.parser.Optional",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('Optional', 'typing.Optional')>",
|
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"abstractmethod": {
|
|
||||||
"name": "abstractmethod",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.html.parser.abstractmethod",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
|
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"BeautifulSoup": {
|
"BeautifulSoup": {
|
||||||
"name": "BeautifulSoup",
|
"name": "BeautifulSoup",
|
||||||
"kind": "alias",
|
"kind": "alias",
|
||||||
@@ -53,43 +46,6 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('Tag', 'bs4.Tag')>",
|
"signature": "<bound method Alias.signature of Alias('Tag', 'bs4.Tag')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
"ContentType": {
|
|
||||||
"name": "ContentType",
|
|
||||||
"kind": "class",
|
|
||||||
"path": "omniread.html.parser.ContentType",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('ContentType', 'omniread.core.content.ContentType')>",
|
|
||||||
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
|
||||||
"members": {
|
|
||||||
"HTML": {
|
|
||||||
"name": "HTML",
|
|
||||||
"kind": "attribute",
|
|
||||||
"path": "omniread.html.parser.ContentType.HTML",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('HTML', 'omniread.core.content.ContentType.HTML')>",
|
|
||||||
"docstring": "HTML document content."
|
|
||||||
},
|
|
||||||
"PDF": {
|
|
||||||
"name": "PDF",
|
|
||||||
"kind": "attribute",
|
|
||||||
"path": "omniread.html.parser.ContentType.PDF",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
|
||||||
"docstring": "PDF document content."
|
|
||||||
},
|
|
||||||
"JSON": {
|
|
||||||
"name": "JSON",
|
|
||||||
"kind": "attribute",
|
|
||||||
"path": "omniread.html.parser.ContentType.JSON",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('JSON', 'omniread.core.content.ContentType.JSON')>",
|
|
||||||
"docstring": "JSON document content."
|
|
||||||
},
|
|
||||||
"XML": {
|
|
||||||
"name": "XML",
|
|
||||||
"kind": "attribute",
|
|
||||||
"path": "omniread.html.parser.ContentType.XML",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('XML', 'omniread.core.content.ContentType.XML')>",
|
|
||||||
"docstring": "XML document content."
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"Content": {
|
"Content": {
|
||||||
"name": "Content",
|
"name": "Content",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
@@ -127,6 +83,57 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.html.parser.ContentType",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('ContentType', 'omniread.core.content.ContentType')>",
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.ContentType.HTML",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('HTML', 'omniread.core.content.ContentType.HTML')>",
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.ContentType.PDF",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.ContentType.XLSX",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XLSX', 'omniread.core.content.ContentType.XLSX')>",
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.ContentType.CSV",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('CSV', 'omniread.core.content.ContentType.CSV')>",
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.ContentType.JSON",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('JSON', 'omniread.core.content.ContentType.JSON')>",
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.parser.ContentType.XML",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XML', 'omniread.core.content.ContentType.XML')>",
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"BaseParser": {
|
"BaseParser": {
|
||||||
"name": "BaseParser",
|
"name": "BaseParser",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
@@ -175,7 +182,7 @@
|
|||||||
"name": "HTMLParser",
|
"name": "HTMLParser",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
"path": "omniread.html.parser.HTMLParser",
|
"path": "omniread.html.parser.HTMLParser",
|
||||||
"signature": "<bound method Class.signature of Class('HTMLParser', 30, 205)>",
|
"signature": "<bound method Class.signature of Class('HTMLParser', 30, 202)>",
|
||||||
"docstring": "Base HTML parser.\n\nNotes:\n **Responsibilities:**\n\n - This class extends the core `BaseParser` with HTML-specific behavior,\n including DOM parsing via BeautifulSoup and reusable extraction helpers.\n - Provides reusable helpers for HTML extraction. Concrete parsers must\n explicitly define the return type.\n\n **Guarantees:**\n\n - Accepts only HTML content.\n - Owns a parsed BeautifulSoup DOM tree.\n - Provides pure helper utilities for common HTML structures.\n\n **Constraints:**\n\n - Concrete subclasses must define the output type `T` and implement\n the `parse()` method.",
|
"docstring": "Base HTML parser.\n\nNotes:\n **Responsibilities:**\n\n - This class extends the core `BaseParser` with HTML-specific behavior,\n including DOM parsing via BeautifulSoup and reusable extraction helpers.\n - Provides reusable helpers for HTML extraction. Concrete parsers must\n explicitly define the return type.\n\n **Guarantees:**\n\n - Accepts only HTML content.\n - Owns a parsed BeautifulSoup DOM tree.\n - Provides pure helper utilities for common HTML structures.\n\n **Constraints:**\n\n - Concrete subclasses must define the output type `T` and implement\n the `parse()` method.",
|
||||||
"members": {
|
"members": {
|
||||||
"supported_types": {
|
"supported_types": {
|
||||||
@@ -204,37 +211,23 @@
|
|||||||
"kind": "function",
|
"kind": "function",
|
||||||
"path": "omniread.html.parser.HTMLParser.parse_link",
|
"path": "omniread.html.parser.HTMLParser.parse_link",
|
||||||
"signature": "<bound method Function.signature of Function('parse_link', 119, 132)>",
|
"signature": "<bound method Function.signature of Function('parse_link', 119, 132)>",
|
||||||
"docstring": "Extract the hyperlink reference from an `<a>` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n Optional[str]:\n The value of the `href` attribute, or None if absent."
|
"docstring": "Extract the hyperlink reference from an `<a>` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n str | None:\n The value of the `href` attribute, or None if absent."
|
||||||
},
|
},
|
||||||
"parse_table": {
|
"parse_table": {
|
||||||
"name": "parse_table",
|
"name": "parse_table",
|
||||||
"kind": "function",
|
"kind": "function",
|
||||||
"path": "omniread.html.parser.HTMLParser.parse_table",
|
"path": "omniread.html.parser.HTMLParser.parse_table",
|
||||||
"signature": "<bound method Function.signature of Function('parse_table', 134, 155)>",
|
"signature": "<bound method Function.signature of Function('parse_table', 134, 152)>",
|
||||||
"docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table (Tag):\n BeautifulSoup tag representing a `<table>`.\n\nReturns:\n list[list[str]]:\n A list of rows, where each row is a list of cell text values."
|
"docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table (Tag):\n BeautifulSoup tag representing a `<table>`.\n\nReturns:\n list[list[str]]:\n A list of rows, where each row is a list of cell text values."
|
||||||
},
|
},
|
||||||
"parse_meta": {
|
"parse_meta": {
|
||||||
"name": "parse_meta",
|
"name": "parse_meta",
|
||||||
"kind": "function",
|
"kind": "function",
|
||||||
"path": "omniread.html.parser.HTMLParser.parse_meta",
|
"path": "omniread.html.parser.HTMLParser.parse_meta",
|
||||||
"signature": "<bound method Function.signature of Function('parse_meta', 177, 205)>",
|
"signature": "<bound method Function.signature of Function('parse_meta', 174, 202)>",
|
||||||
"docstring": "Extract high-level metadata from the HTML document.\n\nReturns:\n dict[str, Any]:\n Dictionary containing extracted metadata.\n\nNotes:\n **Responsibilities:**\n\n - Extract high-level metadata from the HTML document.\n - This includes: Document title, `<meta>` tag name/property to\n content mappings."
|
"docstring": "Extract high-level metadata from the HTML document.\n\nReturns:\n dict[str, Any]:\n Dictionary containing extracted metadata.\n\nNotes:\n **Responsibilities:**\n\n - Extract high-level metadata from the HTML document.\n - This includes: Document title, `<meta>` tag name/property to\n content mappings."
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
|
||||||
"list": {
|
|
||||||
"name": "list",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.html.parser.list",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('list', 'typing.list')>",
|
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"dict": {
|
|
||||||
"name": "dict",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.html.parser.dict",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('dict', 'typing.dict')>",
|
|
||||||
"docstring": null
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -4,11 +4,11 @@
|
|||||||
"path": "omniread.html.scraper",
|
"path": "omniread.html.scraper",
|
||||||
"docstring": "# Summary\n\nHTML scraping implementation for OmniRead.\n\nThis module provides an HTTP-based scraper for retrieving HTML documents.\nIt implements the core `BaseScraper` contract using `httpx` as the transport\nlayer.\n\nThis scraper is responsible for:\n\n- Fetching raw HTML bytes over HTTP(S)\n- Validating response content type\n- Attaching HTTP metadata to the returned content\n\nThis scraper is not responsible for:\n\n- Parsing or interpreting HTML\n- Retrying failed requests\n- Managing crawl policies or rate limiting",
|
"docstring": "# Summary\n\nHTML scraping implementation for OmniRead.\n\nThis module provides an HTTP-based scraper for retrieving HTML documents.\nIt implements the core `BaseScraper` contract using `httpx` as the transport\nlayer.\n\nThis scraper is responsible for:\n\n- Fetching raw HTML bytes over HTTP(S)\n- Validating response content type\n- Attaching HTTP metadata to the returned content\n\nThis scraper is not responsible for:\n\n- Parsing or interpreting HTML\n- Retrying failed requests\n- Managing crawl policies or rate limiting",
|
||||||
"objects": {
|
"objects": {
|
||||||
"httpx": {
|
"Mapping": {
|
||||||
"name": "httpx",
|
"name": "Mapping",
|
||||||
"kind": "alias",
|
"kind": "alias",
|
||||||
"path": "omniread.html.scraper.httpx",
|
"path": "omniread.html.scraper.Mapping",
|
||||||
"signature": "<bound method Alias.signature of Alias('httpx', 'httpx')>",
|
"signature": "<bound method Alias.signature of Alias('Mapping', 'collections.abc.Mapping')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
"Any": {
|
"Any": {
|
||||||
@@ -18,18 +18,11 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
"Mapping": {
|
"httpx": {
|
||||||
"name": "Mapping",
|
"name": "httpx",
|
||||||
"kind": "alias",
|
"kind": "alias",
|
||||||
"path": "omniread.html.scraper.Mapping",
|
"path": "omniread.html.scraper.httpx",
|
||||||
"signature": "<bound method Alias.signature of Alias('Mapping', 'typing.Mapping')>",
|
"signature": "<bound method Alias.signature of Alias('httpx', 'httpx')>",
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"Optional": {
|
|
||||||
"name": "Optional",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.html.scraper.Optional",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('Optional', 'typing.Optional')>",
|
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
"Content": {
|
"Content": {
|
||||||
@@ -90,6 +83,20 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
||||||
"docstring": "PDF document content."
|
"docstring": "PDF document content."
|
||||||
},
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.ContentType.XLSX",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XLSX', 'omniread.core.content.ContentType.XLSX')>",
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.html.scraper.ContentType.CSV",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('CSV', 'omniread.core.content.ContentType.CSV')>",
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
"JSON": {
|
"JSON": {
|
||||||
"name": "JSON",
|
"name": "JSON",
|
||||||
"kind": "attribute",
|
"kind": "attribute",
|
||||||
@@ -118,7 +125,7 @@
|
|||||||
"kind": "function",
|
"kind": "function",
|
||||||
"path": "omniread.html.scraper.BaseScraper.fetch",
|
"path": "omniread.html.scraper.BaseScraper.fetch",
|
||||||
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.core.scraper.BaseScraper.fetch')>",
|
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.core.scraper.BaseScraper.fetch')>",
|
||||||
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
|
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -126,7 +133,7 @@
|
|||||||
"name": "HTMLScraper",
|
"name": "HTMLScraper",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
"path": "omniread.html.scraper.HTMLScraper",
|
"path": "omniread.html.scraper.HTMLScraper",
|
||||||
"signature": "<bound method Class.signature of Class('HTMLScraper', 30, 143)>",
|
"signature": "<bound method Class.signature of Class('HTMLScraper', 32, 143)>",
|
||||||
"docstring": "Base HTML scraper using `httpx`.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns\n them as raw content wrapped in a `Content` object.\n - Fetches raw bytes and metadata only.\n - The scraper uses `httpx.Client` for HTTP requests, enforces an\n HTML content type, and preserves HTTP response metadata.\n\n **Constraints:**\n\n - The scraper does not: Parse HTML, perform retries or backoff,\n handle non-HTML responses.",
|
"docstring": "Base HTML scraper using `httpx`.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns\n them as raw content wrapped in a `Content` object.\n - Fetches raw bytes and metadata only.\n - The scraper uses `httpx.Client` for HTTP requests, enforces an\n HTML content type, and preserves HTTP response metadata.\n\n **Constraints:**\n\n - The scraper does not: Parse HTML, perform retries or backoff,\n handle non-HTML responses.",
|
||||||
"members": {
|
"members": {
|
||||||
"content_type": {
|
"content_type": {
|
||||||
@@ -140,7 +147,7 @@
|
|||||||
"name": "validate_content_type",
|
"name": "validate_content_type",
|
||||||
"kind": "function",
|
"kind": "function",
|
||||||
"path": "omniread.html.scraper.HTMLScraper.validate_content_type",
|
"path": "omniread.html.scraper.HTMLScraper.validate_content_type",
|
||||||
"signature": "<bound method Function.signature of Function('validate_content_type', 78, 102)>",
|
"signature": "<bound method Function.signature of Function('validate_content_type', 80, 102)>",
|
||||||
"docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content."
|
"docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content."
|
||||||
},
|
},
|
||||||
"fetch": {
|
"fetch": {
|
||||||
@@ -148,7 +155,7 @@
|
|||||||
"kind": "function",
|
"kind": "function",
|
||||||
"path": "omniread.html.scraper.HTMLScraper.fetch",
|
"path": "omniread.html.scraper.HTMLScraper.fetch",
|
||||||
"signature": "<bound method Function.signature of Function('fetch', 104, 143)>",
|
"signature": "<bound method Function.signature of Function('fetch', 104, 143)>",
|
||||||
"docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML."
|
"docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML."
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -4,13 +4,6 @@
|
|||||||
"path": "omniread.pdf.client",
|
"path": "omniread.pdf.client",
|
||||||
"docstring": "# Summary\n\nPDF client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw PDF\nbytes from a concrete backing store.\n\nClients provide low-level access to PDF binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems",
|
"docstring": "# Summary\n\nPDF client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw PDF\nbytes from a concrete backing store.\n\nClients provide low-level access to PDF binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems",
|
||||||
"objects": {
|
"objects": {
|
||||||
"Any": {
|
|
||||||
"name": "Any",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.pdf.client.Any",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"ABC": {
|
"ABC": {
|
||||||
"name": "ABC",
|
"name": "ABC",
|
||||||
"kind": "alias",
|
"kind": "alias",
|
||||||
@@ -32,6 +25,13 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('Path', 'pathlib.Path')>",
|
"signature": "<bound method Alias.signature of Alias('Path', 'pathlib.Path')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
|
"Any": {
|
||||||
|
"name": "Any",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.pdf.client.Any",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
"BasePDFClient": {
|
"BasePDFClient": {
|
||||||
"name": "BasePDFClient",
|
"name": "BasePDFClient",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
@@ -32,7 +32,7 @@
|
|||||||
"kind": "function",
|
"kind": "function",
|
||||||
"path": "omniread.pdf.PDFScraper.fetch",
|
"path": "omniread.pdf.PDFScraper.fetch",
|
||||||
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.pdf.scraper.PDFScraper.fetch')>",
|
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.pdf.scraper.PDFScraper.fetch')>",
|
||||||
"docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client."
|
"docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client."
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -66,13 +66,6 @@
|
|||||||
"signature": null,
|
"signature": null,
|
||||||
"docstring": "# Summary\n\nPDF client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw PDF\nbytes from a concrete backing store.\n\nClients provide low-level access to PDF binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems",
|
"docstring": "# Summary\n\nPDF client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw PDF\nbytes from a concrete backing store.\n\nClients provide low-level access to PDF binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems",
|
||||||
"members": {
|
"members": {
|
||||||
"Any": {
|
|
||||||
"name": "Any",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.pdf.client.Any",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"ABC": {
|
"ABC": {
|
||||||
"name": "ABC",
|
"name": "ABC",
|
||||||
"kind": "alias",
|
"kind": "alias",
|
||||||
@@ -94,6 +87,13 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('Path', 'pathlib.Path')>",
|
"signature": "<bound method Alias.signature of Alias('Path', 'pathlib.Path')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
|
"Any": {
|
||||||
|
"name": "Any",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.pdf.client.Any",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
"BasePDFClient": {
|
"BasePDFClient": {
|
||||||
"name": "BasePDFClient",
|
"name": "BasePDFClient",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
@@ -135,6 +135,13 @@
|
|||||||
"signature": null,
|
"signature": null,
|
||||||
"docstring": "# Summary\n\nPDF parser base implementations for OmniRead.\n\nThis module defines the **PDF-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for PDF content.\n\nPDF parsers are responsible for interpreting binary PDF data and producing\nstructured representations suitable for downstream consumption.",
|
"docstring": "# Summary\n\nPDF parser base implementations for OmniRead.\n\nThis module defines the **PDF-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for PDF content.\n\nPDF parsers are responsible for interpreting binary PDF data and producing\nstructured representations suitable for downstream consumption.",
|
||||||
"members": {
|
"members": {
|
||||||
|
"abstractmethod": {
|
||||||
|
"name": "abstractmethod",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.pdf.parser.abstractmethod",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
"Generic": {
|
"Generic": {
|
||||||
"name": "Generic",
|
"name": "Generic",
|
||||||
"kind": "alias",
|
"kind": "alias",
|
||||||
@@ -149,13 +156,6 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('TypeVar', 'typing.TypeVar')>",
|
"signature": "<bound method Alias.signature of Alias('TypeVar', 'typing.TypeVar')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
"abstractmethod": {
|
|
||||||
"name": "abstractmethod",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.pdf.parser.abstractmethod",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
|
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"ContentType": {
|
"ContentType": {
|
||||||
"name": "ContentType",
|
"name": "ContentType",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
@@ -177,6 +177,20 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
||||||
"docstring": "PDF document content."
|
"docstring": "PDF document content."
|
||||||
},
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.parser.ContentType.XLSX",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XLSX', 'omniread.core.content.ContentType.XLSX')>",
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.parser.ContentType.CSV",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('CSV', 'omniread.core.content.ContentType.CSV')>",
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
"JSON": {
|
"JSON": {
|
||||||
"name": "JSON",
|
"name": "JSON",
|
||||||
"kind": "attribute",
|
"kind": "attribute",
|
||||||
@@ -269,6 +283,13 @@
|
|||||||
"signature": null,
|
"signature": null,
|
||||||
"docstring": "# Summary\n\nPDF scraping implementation for OmniRead.\n\nThis module provides a PDF-specific scraper that coordinates PDF byte\nretrieval via a client and normalizes the result into a `Content` object.\n\nThe scraper implements the core `BaseScraper` contract while delegating\nall storage and access concerns to a `BasePDFClient` implementation.",
|
"docstring": "# Summary\n\nPDF scraping implementation for OmniRead.\n\nThis module provides a PDF-specific scraper that coordinates PDF byte\nretrieval via a client and normalizes the result into a `Content` object.\n\nThe scraper implements the core `BaseScraper` contract while delegating\nall storage and access concerns to a `BasePDFClient` implementation.",
|
||||||
"members": {
|
"members": {
|
||||||
|
"Mapping": {
|
||||||
|
"name": "Mapping",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.pdf.scraper.Mapping",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Mapping', 'collections.abc.Mapping')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
"Any": {
|
"Any": {
|
||||||
"name": "Any",
|
"name": "Any",
|
||||||
"kind": "alias",
|
"kind": "alias",
|
||||||
@@ -276,20 +297,6 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
"Mapping": {
|
|
||||||
"name": "Mapping",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.pdf.scraper.Mapping",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('Mapping', 'typing.Mapping')>",
|
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"Optional": {
|
|
||||||
"name": "Optional",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.pdf.scraper.Optional",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('Optional', 'typing.Optional')>",
|
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"Content": {
|
"Content": {
|
||||||
"name": "Content",
|
"name": "Content",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
@@ -348,6 +355,20 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
||||||
"docstring": "PDF document content."
|
"docstring": "PDF document content."
|
||||||
},
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.scraper.ContentType.XLSX",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XLSX', 'omniread.core.content.ContentType.XLSX')>",
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.scraper.ContentType.CSV",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('CSV', 'omniread.core.content.ContentType.CSV')>",
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
"JSON": {
|
"JSON": {
|
||||||
"name": "JSON",
|
"name": "JSON",
|
||||||
"kind": "attribute",
|
"kind": "attribute",
|
||||||
@@ -376,7 +397,7 @@
|
|||||||
"kind": "function",
|
"kind": "function",
|
||||||
"path": "omniread.pdf.scraper.BaseScraper.fetch",
|
"path": "omniread.pdf.scraper.BaseScraper.fetch",
|
||||||
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.core.scraper.BaseScraper.fetch')>",
|
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.core.scraper.BaseScraper.fetch')>",
|
||||||
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
|
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -400,15 +421,15 @@
|
|||||||
"name": "PDFScraper",
|
"name": "PDFScraper",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
"path": "omniread.pdf.scraper.PDFScraper",
|
"path": "omniread.pdf.scraper.PDFScraper",
|
||||||
"signature": "<bound method Class.signature of Class('PDFScraper', 20, 77)>",
|
"signature": "<bound method Class.signature of Class('PDFScraper', 21, 78)>",
|
||||||
"docstring": "Scraper for PDF sources.\n\nNotes:\n **Responsibilities:**\n\n - Delegates byte retrieval to a PDF client and normalizes output\n into `Content`.\n - Preserves caller-provided metadata.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
|
"docstring": "Scraper for PDF sources.\n\nNotes:\n **Responsibilities:**\n\n - Delegates byte retrieval to a PDF client and normalizes output\n into `Content`.\n - Preserves caller-provided metadata.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
|
||||||
"members": {
|
"members": {
|
||||||
"fetch": {
|
"fetch": {
|
||||||
"name": "fetch",
|
"name": "fetch",
|
||||||
"kind": "function",
|
"kind": "function",
|
||||||
"path": "omniread.pdf.scraper.PDFScraper.fetch",
|
"path": "omniread.pdf.scraper.PDFScraper.fetch",
|
||||||
"signature": "<bound method Function.signature of Function('fetch', 47, 77)>",
|
"signature": "<bound method Function.signature of Function('fetch', 48, 78)>",
|
||||||
"docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client."
|
"docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client."
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -4,6 +4,13 @@
|
|||||||
"path": "omniread.pdf.parser",
|
"path": "omniread.pdf.parser",
|
||||||
"docstring": "# Summary\n\nPDF parser base implementations for OmniRead.\n\nThis module defines the **PDF-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for PDF content.\n\nPDF parsers are responsible for interpreting binary PDF data and producing\nstructured representations suitable for downstream consumption.",
|
"docstring": "# Summary\n\nPDF parser base implementations for OmniRead.\n\nThis module defines the **PDF-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for PDF content.\n\nPDF parsers are responsible for interpreting binary PDF data and producing\nstructured representations suitable for downstream consumption.",
|
||||||
"objects": {
|
"objects": {
|
||||||
|
"abstractmethod": {
|
||||||
|
"name": "abstractmethod",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.pdf.parser.abstractmethod",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
"Generic": {
|
"Generic": {
|
||||||
"name": "Generic",
|
"name": "Generic",
|
||||||
"kind": "alias",
|
"kind": "alias",
|
||||||
@@ -18,13 +25,6 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('TypeVar', 'typing.TypeVar')>",
|
"signature": "<bound method Alias.signature of Alias('TypeVar', 'typing.TypeVar')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
"abstractmethod": {
|
|
||||||
"name": "abstractmethod",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.pdf.parser.abstractmethod",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
|
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"ContentType": {
|
"ContentType": {
|
||||||
"name": "ContentType",
|
"name": "ContentType",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
@@ -46,6 +46,20 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
||||||
"docstring": "PDF document content."
|
"docstring": "PDF document content."
|
||||||
},
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.parser.ContentType.XLSX",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XLSX', 'omniread.core.content.ContentType.XLSX')>",
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.parser.ContentType.CSV",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('CSV', 'omniread.core.content.ContentType.CSV')>",
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
"JSON": {
|
"JSON": {
|
||||||
"name": "JSON",
|
"name": "JSON",
|
||||||
"kind": "attribute",
|
"kind": "attribute",
|
||||||
@@ -4,6 +4,13 @@
|
|||||||
"path": "omniread.pdf.scraper",
|
"path": "omniread.pdf.scraper",
|
||||||
"docstring": "# Summary\n\nPDF scraping implementation for OmniRead.\n\nThis module provides a PDF-specific scraper that coordinates PDF byte\nretrieval via a client and normalizes the result into a `Content` object.\n\nThe scraper implements the core `BaseScraper` contract while delegating\nall storage and access concerns to a `BasePDFClient` implementation.",
|
"docstring": "# Summary\n\nPDF scraping implementation for OmniRead.\n\nThis module provides a PDF-specific scraper that coordinates PDF byte\nretrieval via a client and normalizes the result into a `Content` object.\n\nThe scraper implements the core `BaseScraper` contract while delegating\nall storage and access concerns to a `BasePDFClient` implementation.",
|
||||||
"objects": {
|
"objects": {
|
||||||
|
"Mapping": {
|
||||||
|
"name": "Mapping",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.pdf.scraper.Mapping",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Mapping', 'collections.abc.Mapping')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
"Any": {
|
"Any": {
|
||||||
"name": "Any",
|
"name": "Any",
|
||||||
"kind": "alias",
|
"kind": "alias",
|
||||||
@@ -11,20 +18,6 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
||||||
"docstring": null
|
"docstring": null
|
||||||
},
|
},
|
||||||
"Mapping": {
|
|
||||||
"name": "Mapping",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.pdf.scraper.Mapping",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('Mapping', 'typing.Mapping')>",
|
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"Optional": {
|
|
||||||
"name": "Optional",
|
|
||||||
"kind": "alias",
|
|
||||||
"path": "omniread.pdf.scraper.Optional",
|
|
||||||
"signature": "<bound method Alias.signature of Alias('Optional', 'typing.Optional')>",
|
|
||||||
"docstring": null
|
|
||||||
},
|
|
||||||
"Content": {
|
"Content": {
|
||||||
"name": "Content",
|
"name": "Content",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
@@ -83,6 +76,20 @@
|
|||||||
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
||||||
"docstring": "PDF document content."
|
"docstring": "PDF document content."
|
||||||
},
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.scraper.ContentType.XLSX",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XLSX', 'omniread.core.content.ContentType.XLSX')>",
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.pdf.scraper.ContentType.CSV",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('CSV', 'omniread.core.content.ContentType.CSV')>",
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
"JSON": {
|
"JSON": {
|
||||||
"name": "JSON",
|
"name": "JSON",
|
||||||
"kind": "attribute",
|
"kind": "attribute",
|
||||||
@@ -111,7 +118,7 @@
|
|||||||
"kind": "function",
|
"kind": "function",
|
||||||
"path": "omniread.pdf.scraper.BaseScraper.fetch",
|
"path": "omniread.pdf.scraper.BaseScraper.fetch",
|
||||||
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.core.scraper.BaseScraper.fetch')>",
|
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.core.scraper.BaseScraper.fetch')>",
|
||||||
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
|
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -135,15 +142,15 @@
|
|||||||
"name": "PDFScraper",
|
"name": "PDFScraper",
|
||||||
"kind": "class",
|
"kind": "class",
|
||||||
"path": "omniread.pdf.scraper.PDFScraper",
|
"path": "omniread.pdf.scraper.PDFScraper",
|
||||||
"signature": "<bound method Class.signature of Class('PDFScraper', 20, 77)>",
|
"signature": "<bound method Class.signature of Class('PDFScraper', 21, 78)>",
|
||||||
"docstring": "Scraper for PDF sources.\n\nNotes:\n **Responsibilities:**\n\n - Delegates byte retrieval to a PDF client and normalizes output\n into `Content`.\n - Preserves caller-provided metadata.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
|
"docstring": "Scraper for PDF sources.\n\nNotes:\n **Responsibilities:**\n\n - Delegates byte retrieval to a PDF client and normalizes output\n into `Content`.\n - Preserves caller-provided metadata.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
|
||||||
"members": {
|
"members": {
|
||||||
"fetch": {
|
"fetch": {
|
||||||
"name": "fetch",
|
"name": "fetch",
|
||||||
"kind": "function",
|
"kind": "function",
|
||||||
"path": "omniread.pdf.scraper.PDFScraper.fetch",
|
"path": "omniread.pdf.scraper.PDFScraper.fetch",
|
||||||
"signature": "<bound method Function.signature of Function('fetch', 47, 77)>",
|
"signature": "<bound method Function.signature of Function('fetch', 48, 78)>",
|
||||||
"docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client."
|
"docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client."
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
69
docs/mcp/modules/omniread.xlsx.client.json
Normal file
69
docs/mcp/modules/omniread.xlsx.client.json
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.xlsx.client",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.xlsx.client",
|
||||||
|
"docstring": "# Summary\n\nXLSX client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw\nOffice Open XML spreadsheet bytes from a concrete backing store.\n\nClients provide low-level access to xlsx binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems",
|
||||||
|
"objects": {
|
||||||
|
"ABC": {
|
||||||
|
"name": "ABC",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.client.ABC",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('ABC', 'abc.ABC')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"abstractmethod": {
|
||||||
|
"name": "abstractmethod",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.client.abstractmethod",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Path": {
|
||||||
|
"name": "Path",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.client.Path",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Path', 'pathlib.Path')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Any": {
|
||||||
|
"name": "Any",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.client.Any",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"BaseXlsxClient": {
|
||||||
|
"name": "BaseXlsxClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.client.BaseXlsxClient",
|
||||||
|
"signature": "<bound method Class.signature of Class('BaseXlsxClient', 25, 58)>",
|
||||||
|
"docstring": "Abstract client responsible for retrieving spreadsheet bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full xlsx binary payload.\n - Raise retrieval-specific errors on failure.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.client.BaseXlsxClient.fetch",
|
||||||
|
"signature": "<bound method Function.signature of Function('fetch', 40, 58)>",
|
||||||
|
"docstring": "Fetch raw xlsx bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"FileSystemXlsxClient": {
|
||||||
|
"name": "FileSystemXlsxClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.client.FileSystemXlsxClient",
|
||||||
|
"signature": "<bound method Class.signature of Class('FileSystemXlsxClient', 61, 97)>",
|
||||||
|
"docstring": "XLSX client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads spreadsheet files directly from the disk and\n returns their raw binary contents.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.client.FileSystemXlsxClient.fetch",
|
||||||
|
"signature": "<bound method Function.signature of Function('fetch', 72, 97)>",
|
||||||
|
"docstring": "Read an xlsx file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the spreadsheet file.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
611
docs/mcp/modules/omniread.xlsx.json
Normal file
611
docs/mcp/modules/omniread.xlsx.json
Normal file
@@ -0,0 +1,611 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.xlsx",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.xlsx",
|
||||||
|
"docstring": "# Summary\n\nXLSX subpackage for OmniRead.\n\nProvides acquisition and parsing of Office Open XML spreadsheet (xlsx)\ncontent:\n\n- `BaseXlsxClient`: abstract backing-store client for xlsx bytes.\n- `FileSystemXlsxClient`: local filesystem implementation.\n- `XlsxScraper`: wraps fetched bytes into canonical `Content`.\n- `XlsxParserBase`: content-type-enforcing parser contract.\n- `XlsxParser`: generic string-row parser built on openpyxl.",
|
||||||
|
"objects": {
|
||||||
|
"BaseXlsxClient": {
|
||||||
|
"name": "BaseXlsxClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.BaseXlsxClient",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('BaseXlsxClient', 'omniread.xlsx.client.BaseXlsxClient')>",
|
||||||
|
"docstring": "Abstract client responsible for retrieving spreadsheet bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full xlsx binary payload.\n - Raise retrieval-specific errors on failure.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.BaseXlsxClient.fetch",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.xlsx.client.BaseXlsxClient.fetch')>",
|
||||||
|
"docstring": "Fetch raw xlsx bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"FileSystemXlsxClient": {
|
||||||
|
"name": "FileSystemXlsxClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.FileSystemXlsxClient",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('FileSystemXlsxClient', 'omniread.xlsx.client.FileSystemXlsxClient')>",
|
||||||
|
"docstring": "XLSX client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads spreadsheet files directly from the disk and\n returns their raw binary contents.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.FileSystemXlsxClient.fetch",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.xlsx.client.FileSystemXlsxClient.fetch')>",
|
||||||
|
"docstring": "Read an xlsx file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the spreadsheet file.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"XlsxParser": {
|
||||||
|
"name": "XlsxParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.XlsxParser",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XlsxParser', 'omniread.xlsx.parser.XlsxParser')>",
|
||||||
|
"docstring": "Generic xlsx parser producing string rows from a worksheet.\n\nNotes:\n **Responsibilities:**\n\n - Lazily load the workbook owned by the parser's content.\n - Normalize cells (including dates and numeric values) into\n deterministic string representations.\n - Expose sheet discovery and row extraction helpers.\n\n **Constraints:**\n\n - Cells are rendered with ``str(value)`` after trimming; date and\n datetime values are rendered in ISO format. Consumers requiring\n locale-specific formatting must convert on their side.",
|
||||||
|
"members": {
|
||||||
|
"workbook": {
|
||||||
|
"name": "workbook",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.XlsxParser.workbook",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('workbook', 'omniread.xlsx.parser.XlsxParser.workbook')>",
|
||||||
|
"docstring": "The lazily loaded workbook backing this parser's content."
|
||||||
|
},
|
||||||
|
"sheet_names": {
|
||||||
|
"name": "sheet_names",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.XlsxParser.sheet_names",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('sheet_names', 'omniread.xlsx.parser.XlsxParser.sheet_names')>",
|
||||||
|
"docstring": "Names of all worksheets contained in the workbook."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.XlsxParser.parse",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('parse', 'omniread.xlsx.parser.XlsxParser.parse')>",
|
||||||
|
"docstring": "Parse the first worksheet into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the default (first) worksheet."
|
||||||
|
},
|
||||||
|
"rows": {
|
||||||
|
"name": "rows",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.XlsxParser.rows",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('rows', 'omniread.xlsx.parser.XlsxParser.rows')>",
|
||||||
|
"docstring": "Extract normalized string rows from a worksheet.\n\nArgs:\n sheet (int | str | None):\n Worksheet index or title; defaults to the first worksheet.\n skip_empty (bool):\n When True (default), rows whose cells are all blank are omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row.\n\nRaises:\n ValueError:\n If the requested sheet does not exist."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"XlsxParserBase": {
|
||||||
|
"name": "XlsxParserBase",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.XlsxParserBase",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XlsxParserBase', 'omniread.xlsx.parser_base.XlsxParserBase')>",
|
||||||
|
"docstring": "Base xlsx parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces xlsx content-type compatibility and provides\n the extension point for implementing concrete xlsx parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.XlsxParserBase.supported_types",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('supported_types', 'omniread.xlsx.parser_base.XlsxParserBase.supported_types')>",
|
||||||
|
"docstring": "Set of content types supported by this parser (XLSX only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.XlsxParserBase.parse",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('parse', 'omniread.xlsx.parser_base.XlsxParserBase.parse')>",
|
||||||
|
"docstring": "Parse xlsx content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"XlsxScraper": {
|
||||||
|
"name": "XlsxScraper",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.XlsxScraper",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XlsxScraper', 'omniread.xlsx.scraper.XlsxScraper')>",
|
||||||
|
"docstring": "Scraper for xlsx spreadsheet documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw xlsx bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n XLSX content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.XlsxScraper.fetch",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.xlsx.scraper.XlsxScraper.fetch')>",
|
||||||
|
"docstring": "Fetch an xlsx document from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw xlsx bytes, source\n identifier, XLSX content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"client": {
|
||||||
|
"name": "client",
|
||||||
|
"kind": "module",
|
||||||
|
"path": "omniread.xlsx.client",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "# Summary\n\nXLSX client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw\nOffice Open XML spreadsheet bytes from a concrete backing store.\n\nClients provide low-level access to xlsx binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems",
|
||||||
|
"members": {
|
||||||
|
"ABC": {
|
||||||
|
"name": "ABC",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.client.ABC",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('ABC', 'abc.ABC')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"abstractmethod": {
|
||||||
|
"name": "abstractmethod",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.client.abstractmethod",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Path": {
|
||||||
|
"name": "Path",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.client.Path",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Path', 'pathlib.Path')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Any": {
|
||||||
|
"name": "Any",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.client.Any",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"BaseXlsxClient": {
|
||||||
|
"name": "BaseXlsxClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.client.BaseXlsxClient",
|
||||||
|
"signature": "<bound method Class.signature of Class('BaseXlsxClient', 25, 58)>",
|
||||||
|
"docstring": "Abstract client responsible for retrieving spreadsheet bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full xlsx binary payload.\n - Raise retrieval-specific errors on failure.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.client.BaseXlsxClient.fetch",
|
||||||
|
"signature": "<bound method Function.signature of Function('fetch', 40, 58)>",
|
||||||
|
"docstring": "Fetch raw xlsx bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"FileSystemXlsxClient": {
|
||||||
|
"name": "FileSystemXlsxClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.client.FileSystemXlsxClient",
|
||||||
|
"signature": "<bound method Class.signature of Class('FileSystemXlsxClient', 61, 97)>",
|
||||||
|
"docstring": "XLSX client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads spreadsheet files directly from the disk and\n returns their raw binary contents.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.client.FileSystemXlsxClient.fetch",
|
||||||
|
"signature": "<bound method Function.signature of Function('fetch', 72, 97)>",
|
||||||
|
"docstring": "Read an xlsx file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the spreadsheet file.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"parser": {
|
||||||
|
"name": "parser",
|
||||||
|
"kind": "module",
|
||||||
|
"path": "omniread.xlsx.parser",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "# Summary\n\nXLSX parser implementations for OmniRead.\n\nThis module provides a concrete, generic parser for Office Open XML\nspreadsheets. It exposes workbook sheets as lists of string rows so\ndownstream consumers can interpret tabular content without depending on\nopenpyxl directly.\n\nThe parser is intentionally statement-agnostic: it performs no header\ndetection or column interpretation beyond basic cell normalization.",
|
||||||
|
"members": {
|
||||||
|
"datetime": {
|
||||||
|
"name": "datetime",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.parser.datetime",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('datetime', 'datetime')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"BytesIO": {
|
||||||
|
"name": "BytesIO",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.parser.BytesIO",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('BytesIO', 'io.BytesIO')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Any": {
|
||||||
|
"name": "Any",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.parser.Any",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"openpyxl": {
|
||||||
|
"name": "openpyxl",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.parser.openpyxl",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('openpyxl', 'openpyxl')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.parser.Content",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Content', 'omniread.core.content.Content')>",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.Content.raw",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('raw', 'omniread.core.content.Content.raw')>",
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.Content.source",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('source', 'omniread.core.content.Content.source')>",
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.Content.content_type",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('content_type', 'omniread.core.content.Content.content_type')>",
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.Content.metadata",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('metadata', 'omniread.core.content.Content.metadata')>",
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"XlsxParserBase": {
|
||||||
|
"name": "XlsxParserBase",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParserBase",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XlsxParserBase', 'omniread.xlsx.parser_base.XlsxParserBase')>",
|
||||||
|
"docstring": "Base xlsx parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces xlsx content-type compatibility and provides\n the extension point for implementing concrete xlsx parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParserBase.supported_types",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('supported_types', 'omniread.xlsx.parser_base.XlsxParserBase.supported_types')>",
|
||||||
|
"docstring": "Set of content types supported by this parser (XLSX only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParserBase.parse",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('parse', 'omniread.xlsx.parser_base.XlsxParserBase.parse')>",
|
||||||
|
"docstring": "Parse xlsx content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"XlsxParser": {
|
||||||
|
"name": "XlsxParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParser",
|
||||||
|
"signature": "<bound method Class.signature of Class('XlsxParser', 26, 149)>",
|
||||||
|
"docstring": "Generic xlsx parser producing string rows from a worksheet.\n\nNotes:\n **Responsibilities:**\n\n - Lazily load the workbook owned by the parser's content.\n - Normalize cells (including dates and numeric values) into\n deterministic string representations.\n - Expose sheet discovery and row extraction helpers.\n\n **Constraints:**\n\n - Cells are rendered with ``str(value)`` after trimming; date and\n datetime values are rendered in ISO format. Consumers requiring\n locale-specific formatting must convert on their side.",
|
||||||
|
"members": {
|
||||||
|
"workbook": {
|
||||||
|
"name": "workbook",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParser.workbook",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "The lazily loaded workbook backing this parser's content."
|
||||||
|
},
|
||||||
|
"sheet_names": {
|
||||||
|
"name": "sheet_names",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParser.sheet_names",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Names of all worksheets contained in the workbook."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParser.parse",
|
||||||
|
"signature": "<bound method Function.signature of Function('parse', 85, 93)>",
|
||||||
|
"docstring": "Parse the first worksheet into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the default (first) worksheet."
|
||||||
|
},
|
||||||
|
"rows": {
|
||||||
|
"name": "rows",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParser.rows",
|
||||||
|
"signature": "<bound method Function.signature of Function('rows', 95, 127)>",
|
||||||
|
"docstring": "Extract normalized string rows from a worksheet.\n\nArgs:\n sheet (int | str | None):\n Worksheet index or title; defaults to the first worksheet.\n skip_empty (bool):\n When True (default), rows whose cells are all blank are omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row.\n\nRaises:\n ValueError:\n If the requested sheet does not exist."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"parser_base": {
|
||||||
|
"name": "parser_base",
|
||||||
|
"kind": "module",
|
||||||
|
"path": "omniread.xlsx.parser_base",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "# Summary\n\nXLSX parser base implementation for OmniRead.\n\nThis module defines the **XLSX-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for Office Open\nXML spreadsheet content.",
|
||||||
|
"members": {
|
||||||
|
"abstractmethod": {
|
||||||
|
"name": "abstractmethod",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.parser_base.abstractmethod",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Generic": {
|
||||||
|
"name": "Generic",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.parser_base.Generic",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Generic', 'typing.Generic')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"TypeVar": {
|
||||||
|
"name": "TypeVar",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.parser_base.TypeVar",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('TypeVar', 'typing.TypeVar')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('ContentType', 'omniread.core.content.ContentType')>",
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType.HTML",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('HTML', 'omniread.core.content.ContentType.HTML')>",
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType.PDF",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType.XLSX",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XLSX', 'omniread.core.content.ContentType.XLSX')>",
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType.CSV",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('CSV', 'omniread.core.content.ContentType.CSV')>",
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType.JSON",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('JSON', 'omniread.core.content.ContentType.JSON')>",
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType.XML",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XML', 'omniread.core.content.ContentType.XML')>",
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseParser": {
|
||||||
|
"name": "BaseParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.parser_base.BaseParser",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('BaseParser', 'omniread.core.parser.BaseParser')>",
|
||||||
|
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.BaseParser.supported_types",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('supported_types', 'omniread.core.parser.BaseParser.supported_types')>",
|
||||||
|
"docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic."
|
||||||
|
},
|
||||||
|
"content": {
|
||||||
|
"name": "content",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.BaseParser.content",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('content', 'omniread.core.parser.BaseParser.content')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.parser_base.BaseParser.parse",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('parse', 'omniread.core.parser.BaseParser.parse')>",
|
||||||
|
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
|
||||||
|
},
|
||||||
|
"supports": {
|
||||||
|
"name": "supports",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.parser_base.BaseParser.supports",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('supports', 'omniread.core.parser.BaseParser.supports')>",
|
||||||
|
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"T": {
|
||||||
|
"name": "T",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.T",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"XlsxParserBase": {
|
||||||
|
"name": "XlsxParserBase",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.parser_base.XlsxParserBase",
|
||||||
|
"signature": "<bound method Class.signature of Class('XlsxParserBase', 20, 55)>",
|
||||||
|
"docstring": "Base xlsx parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces xlsx content-type compatibility and provides\n the extension point for implementing concrete xlsx parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.XlsxParserBase.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser (XLSX only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.parser_base.XlsxParserBase.parse",
|
||||||
|
"signature": "<bound method Function.signature of Function('parse', 42, 55)>",
|
||||||
|
"docstring": "Parse xlsx content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"scraper": {
|
||||||
|
"name": "scraper",
|
||||||
|
"kind": "module",
|
||||||
|
"path": "omniread.xlsx.scraper",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "# Summary\n\nXLSX scraper for OmniRead.\n\nThis module defines the scraper responsible for acquiring raw Office Open\nXML spreadsheet content from a backing store via a configured client.\n\nThe scraper does not interpret or parse the acquired bytes; it wraps them in\nthe canonical `Content` model.",
|
||||||
|
"members": {
|
||||||
|
"Mapping": {
|
||||||
|
"name": "Mapping",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.scraper.Mapping",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Mapping', 'collections.abc.Mapping')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Any": {
|
||||||
|
"name": "Any",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.scraper.Any",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.scraper.Content",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Content', 'omniread.core.content.Content')>",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.Content.raw",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('raw', 'omniread.core.content.Content.raw')>",
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.Content.source",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('source', 'omniread.core.content.Content.source')>",
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.Content.content_type",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('content_type', 'omniread.core.content.Content.content_type')>",
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.Content.metadata",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('metadata', 'omniread.core.content.Content.metadata')>",
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('ContentType', 'omniread.core.content.ContentType')>",
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType.HTML",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('HTML', 'omniread.core.content.ContentType.HTML')>",
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType.PDF",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType.XLSX",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XLSX', 'omniread.core.content.ContentType.XLSX')>",
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType.CSV",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('CSV', 'omniread.core.content.ContentType.CSV')>",
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType.JSON",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('JSON', 'omniread.core.content.ContentType.JSON')>",
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType.XML",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XML', 'omniread.core.content.ContentType.XML')>",
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseXlsxClient": {
|
||||||
|
"name": "BaseXlsxClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.scraper.BaseXlsxClient",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('BaseXlsxClient', 'omniread.xlsx.client.BaseXlsxClient')>",
|
||||||
|
"docstring": "Abstract client responsible for retrieving spreadsheet bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full xlsx binary payload.\n - Raise retrieval-specific errors on failure.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.scraper.BaseXlsxClient.fetch",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.xlsx.client.BaseXlsxClient.fetch')>",
|
||||||
|
"docstring": "Fetch raw xlsx bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"XlsxScraper": {
|
||||||
|
"name": "XlsxScraper",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.scraper.XlsxScraper",
|
||||||
|
"signature": "<bound method Class.signature of Class('XlsxScraper', 21, 80)>",
|
||||||
|
"docstring": "Scraper for xlsx spreadsheet documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw xlsx bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n XLSX content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.scraper.XlsxScraper.fetch",
|
||||||
|
"signature": "<bound method Function.signature of Function('fetch', 48, 80)>",
|
||||||
|
"docstring": "Fetch an xlsx document from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw xlsx bytes, source\n identifier, XLSX content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
134
docs/mcp/modules/omniread.xlsx.parser.json
Normal file
134
docs/mcp/modules/omniread.xlsx.parser.json
Normal file
@@ -0,0 +1,134 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.xlsx.parser",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.xlsx.parser",
|
||||||
|
"docstring": "# Summary\n\nXLSX parser implementations for OmniRead.\n\nThis module provides a concrete, generic parser for Office Open XML\nspreadsheets. It exposes workbook sheets as lists of string rows so\ndownstream consumers can interpret tabular content without depending on\nopenpyxl directly.\n\nThe parser is intentionally statement-agnostic: it performs no header\ndetection or column interpretation beyond basic cell normalization.",
|
||||||
|
"objects": {
|
||||||
|
"datetime": {
|
||||||
|
"name": "datetime",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.parser.datetime",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('datetime', 'datetime')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"BytesIO": {
|
||||||
|
"name": "BytesIO",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.parser.BytesIO",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('BytesIO', 'io.BytesIO')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Any": {
|
||||||
|
"name": "Any",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.parser.Any",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"openpyxl": {
|
||||||
|
"name": "openpyxl",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.parser.openpyxl",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('openpyxl', 'openpyxl')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.parser.Content",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Content', 'omniread.core.content.Content')>",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.Content.raw",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('raw', 'omniread.core.content.Content.raw')>",
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.Content.source",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('source', 'omniread.core.content.Content.source')>",
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.Content.content_type",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('content_type', 'omniread.core.content.Content.content_type')>",
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.Content.metadata",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('metadata', 'omniread.core.content.Content.metadata')>",
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"XlsxParserBase": {
|
||||||
|
"name": "XlsxParserBase",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParserBase",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XlsxParserBase', 'omniread.xlsx.parser_base.XlsxParserBase')>",
|
||||||
|
"docstring": "Base xlsx parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces xlsx content-type compatibility and provides\n the extension point for implementing concrete xlsx parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParserBase.supported_types",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('supported_types', 'omniread.xlsx.parser_base.XlsxParserBase.supported_types')>",
|
||||||
|
"docstring": "Set of content types supported by this parser (XLSX only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParserBase.parse",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('parse', 'omniread.xlsx.parser_base.XlsxParserBase.parse')>",
|
||||||
|
"docstring": "Parse xlsx content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"XlsxParser": {
|
||||||
|
"name": "XlsxParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParser",
|
||||||
|
"signature": "<bound method Class.signature of Class('XlsxParser', 26, 149)>",
|
||||||
|
"docstring": "Generic xlsx parser producing string rows from a worksheet.\n\nNotes:\n **Responsibilities:**\n\n - Lazily load the workbook owned by the parser's content.\n - Normalize cells (including dates and numeric values) into\n deterministic string representations.\n - Expose sheet discovery and row extraction helpers.\n\n **Constraints:**\n\n - Cells are rendered with ``str(value)`` after trimming; date and\n datetime values are rendered in ISO format. Consumers requiring\n locale-specific formatting must convert on their side.",
|
||||||
|
"members": {
|
||||||
|
"workbook": {
|
||||||
|
"name": "workbook",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParser.workbook",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "The lazily loaded workbook backing this parser's content."
|
||||||
|
},
|
||||||
|
"sheet_names": {
|
||||||
|
"name": "sheet_names",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParser.sheet_names",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Names of all worksheets contained in the workbook."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParser.parse",
|
||||||
|
"signature": "<bound method Function.signature of Function('parse', 85, 93)>",
|
||||||
|
"docstring": "Parse the first worksheet into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the default (first) worksheet."
|
||||||
|
},
|
||||||
|
"rows": {
|
||||||
|
"name": "rows",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.parser.XlsxParser.rows",
|
||||||
|
"signature": "<bound method Function.signature of Function('rows', 95, 127)>",
|
||||||
|
"docstring": "Extract normalized string rows from a worksheet.\n\nArgs:\n sheet (int | str | None):\n Worksheet index or title; defaults to the first worksheet.\n skip_empty (bool):\n When True (default), rows whose cells are all blank are omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row.\n\nRaises:\n ValueError:\n If the requested sheet does not exist."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
148
docs/mcp/modules/omniread.xlsx.parser_base.json
Normal file
148
docs/mcp/modules/omniread.xlsx.parser_base.json
Normal file
@@ -0,0 +1,148 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.xlsx.parser_base",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.xlsx.parser_base",
|
||||||
|
"docstring": "# Summary\n\nXLSX parser base implementation for OmniRead.\n\nThis module defines the **XLSX-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for Office Open\nXML spreadsheet content.",
|
||||||
|
"objects": {
|
||||||
|
"abstractmethod": {
|
||||||
|
"name": "abstractmethod",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.parser_base.abstractmethod",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Generic": {
|
||||||
|
"name": "Generic",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.parser_base.Generic",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Generic', 'typing.Generic')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"TypeVar": {
|
||||||
|
"name": "TypeVar",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.parser_base.TypeVar",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('TypeVar', 'typing.TypeVar')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('ContentType', 'omniread.core.content.ContentType')>",
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType.HTML",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('HTML', 'omniread.core.content.ContentType.HTML')>",
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType.PDF",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType.XLSX",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XLSX', 'omniread.core.content.ContentType.XLSX')>",
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType.CSV",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('CSV', 'omniread.core.content.ContentType.CSV')>",
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType.JSON",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('JSON', 'omniread.core.content.ContentType.JSON')>",
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.ContentType.XML",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XML', 'omniread.core.content.ContentType.XML')>",
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseParser": {
|
||||||
|
"name": "BaseParser",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.parser_base.BaseParser",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('BaseParser', 'omniread.core.parser.BaseParser')>",
|
||||||
|
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.BaseParser.supported_types",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('supported_types', 'omniread.core.parser.BaseParser.supported_types')>",
|
||||||
|
"docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic."
|
||||||
|
},
|
||||||
|
"content": {
|
||||||
|
"name": "content",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.BaseParser.content",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('content', 'omniread.core.parser.BaseParser.content')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.parser_base.BaseParser.parse",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('parse', 'omniread.core.parser.BaseParser.parse')>",
|
||||||
|
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
|
||||||
|
},
|
||||||
|
"supports": {
|
||||||
|
"name": "supports",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.parser_base.BaseParser.supports",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('supports', 'omniread.core.parser.BaseParser.supports')>",
|
||||||
|
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"T": {
|
||||||
|
"name": "T",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.T",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"XlsxParserBase": {
|
||||||
|
"name": "XlsxParserBase",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.parser_base.XlsxParserBase",
|
||||||
|
"signature": "<bound method Class.signature of Class('XlsxParserBase', 20, 55)>",
|
||||||
|
"docstring": "Base xlsx parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces xlsx content-type compatibility and provides\n the extension point for implementing concrete xlsx parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
|
||||||
|
"members": {
|
||||||
|
"supported_types": {
|
||||||
|
"name": "supported_types",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.parser_base.XlsxParserBase.supported_types",
|
||||||
|
"signature": null,
|
||||||
|
"docstring": "Set of content types supported by this parser (XLSX only)."
|
||||||
|
},
|
||||||
|
"parse": {
|
||||||
|
"name": "parse",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.parser_base.XlsxParserBase.parse",
|
||||||
|
"signature": "<bound method Function.signature of Function('parse', 42, 55)>",
|
||||||
|
"docstring": "Parse xlsx content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
143
docs/mcp/modules/omniread.xlsx.scraper.json
Normal file
143
docs/mcp/modules/omniread.xlsx.scraper.json
Normal file
@@ -0,0 +1,143 @@
|
|||||||
|
{
|
||||||
|
"module": "omniread.xlsx.scraper",
|
||||||
|
"content": {
|
||||||
|
"path": "omniread.xlsx.scraper",
|
||||||
|
"docstring": "# Summary\n\nXLSX scraper for OmniRead.\n\nThis module defines the scraper responsible for acquiring raw Office Open\nXML spreadsheet content from a backing store via a configured client.\n\nThe scraper does not interpret or parse the acquired bytes; it wraps them in\nthe canonical `Content` model.",
|
||||||
|
"objects": {
|
||||||
|
"Mapping": {
|
||||||
|
"name": "Mapping",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.scraper.Mapping",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Mapping', 'collections.abc.Mapping')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Any": {
|
||||||
|
"name": "Any",
|
||||||
|
"kind": "alias",
|
||||||
|
"path": "omniread.xlsx.scraper.Any",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
|
||||||
|
"docstring": null
|
||||||
|
},
|
||||||
|
"Content": {
|
||||||
|
"name": "Content",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.scraper.Content",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('Content', 'omniread.core.content.Content')>",
|
||||||
|
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
|
||||||
|
"members": {
|
||||||
|
"raw": {
|
||||||
|
"name": "raw",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.Content.raw",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('raw', 'omniread.core.content.Content.raw')>",
|
||||||
|
"docstring": "Raw content bytes as retrieved from the source."
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"name": "source",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.Content.source",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('source', 'omniread.core.content.Content.source')>",
|
||||||
|
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||||
|
},
|
||||||
|
"content_type": {
|
||||||
|
"name": "content_type",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.Content.content_type",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('content_type', 'omniread.core.content.Content.content_type')>",
|
||||||
|
"docstring": "Optional MIME type of the content, if known."
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"name": "metadata",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.Content.metadata",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('metadata', 'omniread.core.content.Content.metadata')>",
|
||||||
|
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"ContentType": {
|
||||||
|
"name": "ContentType",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('ContentType', 'omniread.core.content.ContentType')>",
|
||||||
|
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
|
||||||
|
"members": {
|
||||||
|
"HTML": {
|
||||||
|
"name": "HTML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType.HTML",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('HTML', 'omniread.core.content.ContentType.HTML')>",
|
||||||
|
"docstring": "HTML document content."
|
||||||
|
},
|
||||||
|
"PDF": {
|
||||||
|
"name": "PDF",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType.PDF",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
|
||||||
|
"docstring": "PDF document content."
|
||||||
|
},
|
||||||
|
"XLSX": {
|
||||||
|
"name": "XLSX",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType.XLSX",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XLSX', 'omniread.core.content.ContentType.XLSX')>",
|
||||||
|
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
|
||||||
|
},
|
||||||
|
"CSV": {
|
||||||
|
"name": "CSV",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType.CSV",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('CSV', 'omniread.core.content.ContentType.CSV')>",
|
||||||
|
"docstring": "Comma-separated-value document content."
|
||||||
|
},
|
||||||
|
"JSON": {
|
||||||
|
"name": "JSON",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType.JSON",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('JSON', 'omniread.core.content.ContentType.JSON')>",
|
||||||
|
"docstring": "JSON document content."
|
||||||
|
},
|
||||||
|
"XML": {
|
||||||
|
"name": "XML",
|
||||||
|
"kind": "attribute",
|
||||||
|
"path": "omniread.xlsx.scraper.ContentType.XML",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('XML', 'omniread.core.content.ContentType.XML')>",
|
||||||
|
"docstring": "XML document content."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"BaseXlsxClient": {
|
||||||
|
"name": "BaseXlsxClient",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.scraper.BaseXlsxClient",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('BaseXlsxClient', 'omniread.xlsx.client.BaseXlsxClient')>",
|
||||||
|
"docstring": "Abstract client responsible for retrieving spreadsheet bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full xlsx binary payload.\n - Raise retrieval-specific errors on failure.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.scraper.BaseXlsxClient.fetch",
|
||||||
|
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.xlsx.client.BaseXlsxClient.fetch')>",
|
||||||
|
"docstring": "Fetch raw xlsx bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"XlsxScraper": {
|
||||||
|
"name": "XlsxScraper",
|
||||||
|
"kind": "class",
|
||||||
|
"path": "omniread.xlsx.scraper.XlsxScraper",
|
||||||
|
"signature": "<bound method Class.signature of Class('XlsxScraper', 21, 80)>",
|
||||||
|
"docstring": "Scraper for xlsx spreadsheet documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw xlsx bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n XLSX content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
|
||||||
|
"members": {
|
||||||
|
"fetch": {
|
||||||
|
"name": "fetch",
|
||||||
|
"kind": "function",
|
||||||
|
"path": "omniread.xlsx.scraper.XlsxScraper.fetch",
|
||||||
|
"signature": "<bound method Function.signature of Function('fetch', 48, 80)>",
|
||||||
|
"docstring": "Fetch an xlsx document from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw xlsx bytes, source\n identifier, XLSX content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
90
docs/mcp/nav.json
Normal file
90
docs/mcp/nav.json
Normal file
@@ -0,0 +1,90 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"module": "omniread",
|
||||||
|
"resource": "docs://modules/omniread"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.core",
|
||||||
|
"resource": "docs://modules/omniread.core"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.core.content",
|
||||||
|
"resource": "docs://modules/omniread.core.content"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.core.parser",
|
||||||
|
"resource": "docs://modules/omniread.core.parser"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.core.scraper",
|
||||||
|
"resource": "docs://modules/omniread.core.scraper"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.csv",
|
||||||
|
"resource": "docs://modules/omniread.csv"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.csv.client",
|
||||||
|
"resource": "docs://modules/omniread.csv.client"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.csv.parser",
|
||||||
|
"resource": "docs://modules/omniread.csv.parser"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.csv.parser_base",
|
||||||
|
"resource": "docs://modules/omniread.csv.parser_base"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.csv.scraper",
|
||||||
|
"resource": "docs://modules/omniread.csv.scraper"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.html",
|
||||||
|
"resource": "docs://modules/omniread.html"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.html.parser",
|
||||||
|
"resource": "docs://modules/omniread.html.parser"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.html.scraper",
|
||||||
|
"resource": "docs://modules/omniread.html.scraper"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.pdf",
|
||||||
|
"resource": "docs://modules/omniread.pdf"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.pdf.client",
|
||||||
|
"resource": "docs://modules/omniread.pdf.client"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.pdf.parser",
|
||||||
|
"resource": "docs://modules/omniread.pdf.parser"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.pdf.scraper",
|
||||||
|
"resource": "docs://modules/omniread.pdf.scraper"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.xlsx",
|
||||||
|
"resource": "docs://modules/omniread.xlsx"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.xlsx.client",
|
||||||
|
"resource": "docs://modules/omniread.xlsx.client"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.xlsx.parser",
|
||||||
|
"resource": "docs://modules/omniread.xlsx.parser"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.xlsx.parser_base",
|
||||||
|
"resource": "docs://modules/omniread.xlsx.parser_base"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"module": "omniread.xlsx.scraper",
|
||||||
|
"resource": "docs://modules/omniread.xlsx.scraper"
|
||||||
|
}
|
||||||
|
]
|
||||||
@@ -1,50 +0,0 @@
|
|||||||
[
|
|
||||||
{
|
|
||||||
"module": "omniread",
|
|
||||||
"resource": "doc://modules/omniread"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"module": "omniread.core",
|
|
||||||
"resource": "doc://modules/omniread.core"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"module": "omniread.core.content",
|
|
||||||
"resource": "doc://modules/omniread.core.content"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"module": "omniread.core.parser",
|
|
||||||
"resource": "doc://modules/omniread.core.parser"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"module": "omniread.core.scraper",
|
|
||||||
"resource": "doc://modules/omniread.core.scraper"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"module": "omniread.html",
|
|
||||||
"resource": "doc://modules/omniread.html"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"module": "omniread.html.parser",
|
|
||||||
"resource": "doc://modules/omniread.html.parser"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"module": "omniread.html.scraper",
|
|
||||||
"resource": "doc://modules/omniread.html.scraper"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"module": "omniread.pdf",
|
|
||||||
"resource": "doc://modules/omniread.pdf"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"module": "omniread.pdf.client",
|
|
||||||
"resource": "doc://modules/omniread.pdf.client"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"module": "omniread.pdf.parser",
|
|
||||||
"resource": "doc://modules/omniread.pdf.parser"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"module": "omniread.pdf.scraper",
|
|
||||||
"resource": "doc://modules/omniread.pdf.scraper"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
71
mkdocs.yml
71
mkdocs.yml
@@ -21,6 +21,31 @@ theme:
|
|||||||
- search.highlight
|
- search.highlight
|
||||||
- search.share
|
- search.share
|
||||||
- search.suggest
|
- search.suggest
|
||||||
|
icon:
|
||||||
|
logo: material/book-open-page-variant
|
||||||
|
repo: fontawesome/brands/github
|
||||||
|
markdown_extensions:
|
||||||
|
- pymdownx.superfences
|
||||||
|
- pymdownx.inlinehilite
|
||||||
|
- pymdownx.snippets
|
||||||
|
- admonition
|
||||||
|
- pymdownx.details
|
||||||
|
- pymdownx.highlight:
|
||||||
|
linenums: true
|
||||||
|
anchor_linenums: true
|
||||||
|
line_spans: __span
|
||||||
|
pygments_lang_class: true
|
||||||
|
- pymdownx.tabbed:
|
||||||
|
alternate_style: true
|
||||||
|
- pymdownx.tasklist:
|
||||||
|
custom_checkbox: true
|
||||||
|
- tables
|
||||||
|
- footnotes
|
||||||
|
- pymdownx.caret
|
||||||
|
- pymdownx.tilde
|
||||||
|
- pymdownx.mark
|
||||||
|
extra_css:
|
||||||
|
- https://unpkg.com/dracula-prism/dist/css/dracula-prism.css
|
||||||
plugins:
|
plugins:
|
||||||
- search
|
- search
|
||||||
- mkdocstrings:
|
- mkdocstrings:
|
||||||
@@ -41,41 +66,21 @@ plugins:
|
|||||||
show_category_heading: true
|
show_category_heading: true
|
||||||
show_object_full_path: false
|
show_object_full_path: false
|
||||||
show_symbol_type_heading: true
|
show_symbol_type_heading: true
|
||||||
markdown_extensions:
|
|
||||||
- pymdownx.superfences
|
|
||||||
- pymdownx.inlinehilite
|
|
||||||
- pymdownx.snippets
|
|
||||||
- admonition
|
|
||||||
- pymdownx.details
|
|
||||||
- pymdownx.superfences
|
|
||||||
- pymdownx.highlight:
|
|
||||||
linenums: true
|
|
||||||
anchor_linenums: true
|
|
||||||
line_spans: __span
|
|
||||||
pygments_lang_class: true
|
|
||||||
- pymdownx.tabbed:
|
|
||||||
alternate_style: true
|
|
||||||
- pymdownx.tasklist:
|
|
||||||
custom_checkbox: true
|
|
||||||
- tables
|
|
||||||
- footnotes
|
|
||||||
- pymdownx.caret
|
|
||||||
- pymdownx.tilde
|
|
||||||
- pymdownx.mark
|
|
||||||
site_name: omniread
|
site_name: omniread
|
||||||
|
docs_dir: docs
|
||||||
nav:
|
nav:
|
||||||
- Home: index.md
|
- Home: lib/index.md
|
||||||
- Core API:
|
- Core API:
|
||||||
- core/index.md
|
- lib/core/index.md
|
||||||
- core/content.md
|
- lib/core/content.md
|
||||||
- core/parser.md
|
- lib/core/parser.md
|
||||||
- core/scraper.md
|
- lib/core/scraper.md
|
||||||
- HTML Handling:
|
- HTML Handling:
|
||||||
- html/index.md
|
- lib/html/index.md
|
||||||
- html/parser.md
|
- lib/html/parser.md
|
||||||
- html/scraper.md
|
- lib/html/scraper.md
|
||||||
- PDF Handling:
|
- PDF Handling:
|
||||||
- pdf/index.md
|
- lib/pdf/index.md
|
||||||
- pdf/client.md
|
- lib/pdf/client.md
|
||||||
- pdf/parser.md
|
- lib/pdf/parser.md
|
||||||
- pdf/scraper.md
|
- lib/pdf/scraper.md
|
||||||
|
|||||||
@@ -132,23 +132,19 @@ __all__ = [
|
|||||||
# core
|
# core
|
||||||
"Content",
|
"Content",
|
||||||
"ContentType",
|
"ContentType",
|
||||||
|
|
||||||
# html
|
# html
|
||||||
"HTMLScraper",
|
"HTMLScraper",
|
||||||
"HTMLParser",
|
"HTMLParser",
|
||||||
|
|
||||||
# pdf
|
# pdf
|
||||||
"FileSystemPDFClient",
|
"FileSystemPDFClient",
|
||||||
"PDFScraper",
|
"PDFScraper",
|
||||||
"PDFParser",
|
"PDFParser",
|
||||||
|
|
||||||
# csv
|
# csv
|
||||||
"BaseCsvClient",
|
"BaseCsvClient",
|
||||||
"FileSystemCsvClient",
|
"FileSystemCsvClient",
|
||||||
"CsvScraper",
|
"CsvScraper",
|
||||||
"CsvParser",
|
"CsvParser",
|
||||||
"CsvParserBase",
|
"CsvParserBase",
|
||||||
|
|
||||||
# xlsx
|
# xlsx
|
||||||
"BaseXlsxClient",
|
"BaseXlsxClient",
|
||||||
"FileSystemXlsxClient",
|
"FileSystemXlsxClient",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
from .core import Content, ContentType
|
from .core import Content, ContentType
|
||||||
from .html import HTMLScraper, HTMLParser
|
from .html import HTMLParser, HTMLScraper
|
||||||
from .pdf import FileSystemPDFClient, PDFScraper, PDFParser
|
from .pdf import FileSystemPDFClient, PDFParser, PDFScraper
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"Content",
|
"Content",
|
||||||
|
|||||||
@@ -11,9 +11,10 @@ retrieved or parsed. Format-specific behavior and metadata must not alter
|
|||||||
the semantic meaning of these models.
|
the semantic meaning of these models.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from enum import Enum
|
from collections.abc import Mapping
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any, Mapping, Optional
|
from enum import Enum
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
class ContentType(str, Enum):
|
class ContentType(str, Enum):
|
||||||
@@ -72,12 +73,12 @@ class Content:
|
|||||||
Identifier of the content origin (URL, file path, or logical name).
|
Identifier of the content origin (URL, file path, or logical name).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
content_type: Optional[ContentType] = None
|
content_type: ContentType | None = None
|
||||||
"""
|
"""
|
||||||
Optional MIME type of the content, if known.
|
Optional MIME type of the content, if known.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
metadata: Optional[Mapping[str, Any]] = None
|
metadata: Mapping[str, Any] | None = None
|
||||||
"""
|
"""
|
||||||
Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes).
|
Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes).
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -1,15 +1,24 @@
|
|||||||
|
from collections.abc import Mapping
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from typing import Any, Mapping, Optional
|
from typing import Any
|
||||||
|
|
||||||
class ContentType(str, Enum):
|
class ContentType(str, Enum):
|
||||||
HTML = "text/html"
|
HTML = "text/html"
|
||||||
PDF = "application/pdf"
|
PDF = "application/pdf"
|
||||||
|
XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||||||
|
CSV = "text/csv"
|
||||||
JSON = "application/json"
|
JSON = "application/json"
|
||||||
XML = "application/xml"
|
XML = "application/xml"
|
||||||
|
|
||||||
class Content:
|
class Content:
|
||||||
raw: bytes
|
raw: bytes
|
||||||
source: str
|
source: str
|
||||||
content_type: Optional[ContentType]
|
content_type: ContentType | None
|
||||||
metadata: Optional[Mapping[str, Any]]
|
metadata: Mapping[str, Any] | None
|
||||||
def __init__(self, raw: bytes, source: str, content_type: Optional[ContentType] = ..., metadata: Optional[Mapping[str, Any]] = ...) -> None: ...
|
def __init__(
|
||||||
|
self,
|
||||||
|
raw: bytes,
|
||||||
|
source: str,
|
||||||
|
content_type: ContentType | None = ...,
|
||||||
|
metadata: Mapping[str, Any] | None = ...,
|
||||||
|
) -> None: ...
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ Parsers are not responsible for:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from typing import Generic, TypeVar, Set
|
from typing import Generic, TypeVar
|
||||||
|
|
||||||
from .content import Content, ContentType
|
from .content import Content, ContentType
|
||||||
|
|
||||||
@@ -46,7 +46,7 @@ class BaseParser(ABC, Generic[T]):
|
|||||||
- Implementations must remain deterministic for a given input.
|
- Implementations must remain deterministic for a given input.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
supported_types: Set[ContentType] = set()
|
supported_types: set[ContentType] = set()
|
||||||
"""
|
"""
|
||||||
Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic.
|
Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic.
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -1,11 +1,12 @@
|
|||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from typing import Generic, TypeVar, Set
|
from typing import Generic, TypeVar
|
||||||
|
|
||||||
from .content import Content, ContentType
|
from .content import Content, ContentType
|
||||||
|
|
||||||
T = TypeVar("T")
|
T = TypeVar("T")
|
||||||
|
|
||||||
class BaseParser(ABC, Generic[T]):
|
class BaseParser(ABC, Generic[T]):
|
||||||
supported_types: Set[ContentType]
|
supported_types: set[ContentType]
|
||||||
content: Content
|
content: Content
|
||||||
def __init__(self, content: Content) -> None: ...
|
def __init__(self, content: Content) -> None: ...
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
|
|||||||
@@ -22,7 +22,8 @@ All interpretation must be delegated to parsers.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from typing import Any, Mapping, Optional
|
from collections.abc import Mapping
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
from .content import Content
|
from .content import Content
|
||||||
|
|
||||||
@@ -53,7 +54,7 @@ class BaseScraper(ABC):
|
|||||||
self,
|
self,
|
||||||
source: str,
|
source: str,
|
||||||
*,
|
*,
|
||||||
metadata: Optional[Mapping[str, Any]] = None,
|
metadata: Mapping[str, Any] | None = None,
|
||||||
) -> Content:
|
) -> Content:
|
||||||
"""
|
"""
|
||||||
Fetch raw content from the given source.
|
Fetch raw content from the given source.
|
||||||
@@ -62,7 +63,7 @@ class BaseScraper(ABC):
|
|||||||
source (str):
|
source (str):
|
||||||
Location identifier (URL, file path, S3 URI, etc.).
|
Location identifier (URL, file path, S3 URI, etc.).
|
||||||
|
|
||||||
metadata (Optional[Mapping[str, Any]], optional):
|
metadata (Mapping[str, Any] | None, optional):
|
||||||
Optional hints for the scraper (headers, auth, etc.).
|
Optional hints for the scraper (headers, auth, etc.).
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
|
|||||||
@@ -1,7 +1,11 @@
|
|||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from typing import Any, Mapping, Optional
|
from collections.abc import Mapping
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
from .content import Content
|
from .content import Content
|
||||||
|
|
||||||
class BaseScraper(ABC):
|
class BaseScraper(ABC):
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def fetch(self, source: str, *, metadata: Optional[Mapping[str, Any]] = ...) -> Content: ...
|
def fetch(
|
||||||
|
self, source: str, *, metadata: Mapping[str, Any] | None = ...
|
||||||
|
) -> Content: ...
|
||||||
|
|||||||
12
omniread/csv/__init__.pyi
Normal file
12
omniread/csv/__init__.pyi
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
from .client import BaseCsvClient, FileSystemCsvClient
|
||||||
|
from .parser import CsvParser
|
||||||
|
from .parser_base import CsvParserBase
|
||||||
|
from .scraper import CsvScraper
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"BaseCsvClient",
|
||||||
|
"FileSystemCsvClient",
|
||||||
|
"CsvScraper",
|
||||||
|
"CsvParser",
|
||||||
|
"CsvParserBase",
|
||||||
|
]
|
||||||
@@ -17,9 +17,9 @@ Typical backing stores include:
|
|||||||
- Network file systems
|
- Network file systems
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Any
|
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
class BaseCsvClient(ABC):
|
class BaseCsvClient(ABC):
|
||||||
|
|||||||
10
omniread/csv/client.pyi
Normal file
10
omniread/csv/client.pyi
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
from abc import ABC, abstractmethod
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
class BaseCsvClient(ABC):
|
||||||
|
@abstractmethod
|
||||||
|
def fetch(self, source: Any) -> bytes: ...
|
||||||
|
|
||||||
|
class FileSystemCsvClient(BaseCsvClient):
|
||||||
|
def fetch(self, path: Path) -> bytes: ...
|
||||||
@@ -13,15 +13,15 @@ detection or column interpretation beyond basic cell normalization and
|
|||||||
delimiter detection.
|
delimiter detection.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from io import StringIO
|
|
||||||
from csv import Sniffer, reader
|
from csv import Sniffer, reader
|
||||||
from typing import List
|
from io import StringIO
|
||||||
|
|
||||||
from omniread.core.content import Content
|
from omniread.core.content import Content
|
||||||
|
|
||||||
from .parser_base import CsvParserBase
|
from .parser_base import CsvParserBase
|
||||||
|
|
||||||
|
|
||||||
class CsvParser(CsvParserBase):
|
class CsvParser(CsvParserBase[list[list[str]]]):
|
||||||
"""
|
"""
|
||||||
Generic csv parser producing string rows from the document.
|
Generic csv parser producing string rows from the document.
|
||||||
|
|
||||||
@@ -54,17 +54,17 @@ class CsvParser(CsvParserBase):
|
|||||||
"""
|
"""
|
||||||
super().__init__(content)
|
super().__init__(content)
|
||||||
|
|
||||||
def parse(self) -> List[List[str]]:
|
def parse(self) -> list[list[str]]:
|
||||||
"""
|
"""
|
||||||
Parse the document into normalized string rows.
|
Parse the document into normalized string rows.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
List[List[str]]:
|
list[list[str]]:
|
||||||
Rows of the document.
|
Rows of the document.
|
||||||
"""
|
"""
|
||||||
return self.rows()
|
return self.rows()
|
||||||
|
|
||||||
def rows(self, *, skip_empty: bool = True) -> List[List[str]]:
|
def rows(self, *, skip_empty: bool = True) -> list[list[str]]:
|
||||||
"""
|
"""
|
||||||
Extract normalized string rows from the document.
|
Extract normalized string rows from the document.
|
||||||
|
|
||||||
@@ -74,7 +74,7 @@ class CsvParser(CsvParserBase):
|
|||||||
omitted.
|
omitted.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
List[List[str]]:
|
list[list[str]]:
|
||||||
Normalized rows; trailing blank cells are trimmed per row.
|
Normalized rows; trailing blank cells are trimmed per row.
|
||||||
"""
|
"""
|
||||||
text = self._decode(self.content.raw)
|
text = self._decode(self.content.raw)
|
||||||
@@ -84,7 +84,7 @@ class CsvParser(CsvParserBase):
|
|||||||
delimiter = dialect.delimiter
|
delimiter = dialect.delimiter
|
||||||
except Exception:
|
except Exception:
|
||||||
delimiter = ","
|
delimiter = ","
|
||||||
out: List[List[str]] = []
|
out: list[list[str]] = []
|
||||||
for row in reader(StringIO(text), delimiter=delimiter):
|
for row in reader(StringIO(text), delimiter=delimiter):
|
||||||
cells = [c.strip() for c in row]
|
cells = [c.strip() for c in row]
|
||||||
while cells and not cells[-1]:
|
while cells and not cells[-1]:
|
||||||
|
|||||||
8
omniread/csv/parser.pyi
Normal file
8
omniread/csv/parser.pyi
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
from omniread.core.content import Content
|
||||||
|
|
||||||
|
from .parser_base import CsvParserBase
|
||||||
|
|
||||||
|
class CsvParser(CsvParserBase[list[list[str]]]):
|
||||||
|
def __init__(self, content: Content) -> None: ...
|
||||||
|
def parse(self) -> list[list[str]]: ...
|
||||||
|
def rows(self, *, skip_empty: bool = True) -> list[list[str]]: ...
|
||||||
@@ -8,8 +8,8 @@ format-agnostic `BaseParser` with constraints appropriate for
|
|||||||
comma-separated-value documents.
|
comma-separated-value documents.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Generic, TypeVar
|
|
||||||
from abc import abstractmethod
|
from abc import abstractmethod
|
||||||
|
from typing import Generic, TypeVar
|
||||||
|
|
||||||
from omniread.core.content import ContentType
|
from omniread.core.content import ContentType
|
||||||
from omniread.core.parser import BaseParser
|
from omniread.core.parser import BaseParser
|
||||||
|
|||||||
13
omniread/csv/parser_base.pyi
Normal file
13
omniread/csv/parser_base.pyi
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
from abc import abstractmethod
|
||||||
|
from typing import Generic, TypeVar
|
||||||
|
|
||||||
|
from omniread.core.content import ContentType
|
||||||
|
from omniread.core.parser import BaseParser
|
||||||
|
|
||||||
|
T = TypeVar("T")
|
||||||
|
|
||||||
|
class CsvParserBase(BaseParser[T], Generic[T]):
|
||||||
|
supported_types: set[ContentType]
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def parse(self) -> T: ...
|
||||||
@@ -11,9 +11,11 @@ The scraper does not interpret or parse the acquired bytes; it wraps them in
|
|||||||
the canonical `Content` model.
|
the canonical `Content` model.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Any, Mapping, Optional
|
from collections.abc import Mapping
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
from omniread.core.content import Content, ContentType
|
from omniread.core.content import Content, ContentType
|
||||||
|
|
||||||
from .client import BaseCsvClient
|
from .client import BaseCsvClient
|
||||||
|
|
||||||
|
|
||||||
@@ -48,7 +50,7 @@ class CsvScraper:
|
|||||||
self,
|
self,
|
||||||
source: Any,
|
source: Any,
|
||||||
*,
|
*,
|
||||||
metadata: Optional[Mapping[str, Any]] = None,
|
metadata: Mapping[str, Any] | None = None,
|
||||||
) -> Content:
|
) -> Content:
|
||||||
"""
|
"""
|
||||||
Fetch a csv document from the given source.
|
Fetch a csv document from the given source.
|
||||||
@@ -57,7 +59,7 @@ class CsvScraper:
|
|||||||
source (Any):
|
source (Any):
|
||||||
Identifier of the csv source as understood by the
|
Identifier of the csv source as understood by the
|
||||||
configured client.
|
configured client.
|
||||||
metadata (Optional[Mapping[str, Any]], optional):
|
metadata (Mapping[str, Any] | None, optional):
|
||||||
Optional metadata to attach to the returned content.
|
Optional metadata to attach to the returned content.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
|
|||||||
12
omniread/csv/scraper.pyi
Normal file
12
omniread/csv/scraper.pyi
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
from collections.abc import Mapping
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from omniread.core.content import Content
|
||||||
|
|
||||||
|
from .client import BaseCsvClient
|
||||||
|
|
||||||
|
class CsvScraper:
|
||||||
|
def __init__(self, *, client: BaseCsvClient) -> None: ...
|
||||||
|
def fetch(
|
||||||
|
self, source: Any, *, metadata: Mapping[str, Any] | None = ...
|
||||||
|
) -> Content: ...
|
||||||
@@ -30,7 +30,6 @@ use this package only when HTML-specific behavior is required.
|
|||||||
---
|
---
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
from .scraper import HTMLScraper
|
from .scraper import HTMLScraper
|
||||||
from .parser import HTMLParser
|
from .parser import HTMLParser
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
from .scraper import HTMLScraper
|
|
||||||
from .parser import HTMLParser
|
from .parser import HTMLParser
|
||||||
|
from .scraper import HTMLScraper
|
||||||
|
|
||||||
__all__ = ["HTMLScraper", "HTMLParser"]
|
__all__ = ["HTMLScraper", "HTMLParser"]
|
||||||
|
|||||||
@@ -16,12 +16,12 @@ Concrete parsers must subclass `HTMLParser` and implement the `parse()` method
|
|||||||
to return a structured representation appropriate for their use case.
|
to return a structured representation appropriate for their use case.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Any, Generic, TypeVar, Optional
|
|
||||||
from abc import abstractmethod
|
from abc import abstractmethod
|
||||||
|
from typing import Any, Generic, TypeVar
|
||||||
|
|
||||||
from bs4 import BeautifulSoup, Tag
|
from bs4 import BeautifulSoup, Tag
|
||||||
|
|
||||||
from omniread.core.content import ContentType, Content
|
from omniread.core.content import Content, ContentType
|
||||||
from omniread.core.parser import BaseParser
|
from omniread.core.parser import BaseParser
|
||||||
|
|
||||||
T = TypeVar("T")
|
T = TypeVar("T")
|
||||||
@@ -117,7 +117,7 @@ class HTMLParser(BaseParser[T], Generic[T]):
|
|||||||
return div.get_text(separator=separator, strip=True)
|
return div.get_text(separator=separator, strip=True)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def parse_link(a: Tag) -> Optional[str]:
|
def parse_link(a: Tag) -> str | None:
|
||||||
"""
|
"""
|
||||||
Extract the hyperlink reference from an `<a>` element.
|
Extract the hyperlink reference from an `<a>` element.
|
||||||
|
|
||||||
@@ -126,7 +126,7 @@ class HTMLParser(BaseParser[T], Generic[T]):
|
|||||||
BeautifulSoup tag representing an anchor.
|
BeautifulSoup tag representing an anchor.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Optional[str]:
|
str | None:
|
||||||
The value of the `href` attribute, or None if absent.
|
The value of the `href` attribute, or None if absent.
|
||||||
"""
|
"""
|
||||||
return a.get("href")
|
return a.get("href")
|
||||||
@@ -146,10 +146,7 @@ class HTMLParser(BaseParser[T], Generic[T]):
|
|||||||
"""
|
"""
|
||||||
rows: list[list[str]] = []
|
rows: list[list[str]] = []
|
||||||
for tr in table.find_all("tr"):
|
for tr in table.find_all("tr"):
|
||||||
cells = [
|
cells = [cell.get_text(strip=True) for cell in tr.find_all(["td", "th"])]
|
||||||
cell.get_text(strip=True)
|
|
||||||
for cell in tr.find_all(["td", "th"])
|
|
||||||
]
|
|
||||||
if cells:
|
if cells:
|
||||||
rows.append(cells)
|
rows.append(cells)
|
||||||
return rows
|
return rows
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
from typing import Any, Generic, TypeVar, Optional, list, dict
|
from typing import Any, Generic, TypeVar
|
||||||
from bs4 import BeautifulSoup, Tag
|
|
||||||
from omniread.core.content import ContentType, Content
|
from bs4 import Tag
|
||||||
|
|
||||||
|
from omniread.core.content import Content, ContentType
|
||||||
from omniread.core.parser import BaseParser
|
from omniread.core.parser import BaseParser
|
||||||
|
|
||||||
T = TypeVar("T")
|
T = TypeVar("T")
|
||||||
@@ -12,7 +14,7 @@ class HTMLParser(BaseParser[T], Generic[T]):
|
|||||||
@staticmethod
|
@staticmethod
|
||||||
def parse_div(div: Tag, *, separator: str = ...) -> str: ...
|
def parse_div(div: Tag, *, separator: str = ...) -> str: ...
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def parse_link(a: Tag) -> Optional[str]: ...
|
def parse_link(a: Tag) -> str | None: ...
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def parse_table(table: Tag) -> list[list[str]]: ...
|
def parse_table(table: Tag) -> list[list[str]]: ...
|
||||||
def parse_meta(self) -> dict[str, Any]: ...
|
def parse_meta(self) -> dict[str, Any]: ...
|
||||||
|
|||||||
@@ -20,8 +20,10 @@ This scraper is not responsible for:
|
|||||||
- Managing crawl policies or rate limiting
|
- Managing crawl policies or rate limiting
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from typing import Any, Mapping, Optional
|
|
||||||
|
|
||||||
from omniread.core.content import Content, ContentType
|
from omniread.core.content import Content, ContentType
|
||||||
from omniread.core.scraper import BaseScraper
|
from omniread.core.scraper import BaseScraper
|
||||||
@@ -51,7 +53,7 @@ class HTMLScraper(BaseScraper):
|
|||||||
*,
|
*,
|
||||||
client: httpx.Client | None = None,
|
client: httpx.Client | None = None,
|
||||||
timeout: float = 15.0,
|
timeout: float = 15.0,
|
||||||
headers: Optional[Mapping[str, str]] = None,
|
headers: Mapping[str, str] | None = None,
|
||||||
follow_redirects: bool = True,
|
follow_redirects: bool = True,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
@@ -62,7 +64,7 @@ class HTMLScraper(BaseScraper):
|
|||||||
Optional pre-configured `httpx.Client`. If omitted, a client is created internally.
|
Optional pre-configured `httpx.Client`. If omitted, a client is created internally.
|
||||||
timeout (float, optional):
|
timeout (float, optional):
|
||||||
Request timeout in seconds.
|
Request timeout in seconds.
|
||||||
headers (Optional[Mapping[str, str]], optional):
|
headers (Mapping[str, str] | None, optional):
|
||||||
Optional default HTTP headers.
|
Optional default HTTP headers.
|
||||||
follow_redirects (bool, optional):
|
follow_redirects (bool, optional):
|
||||||
Whether to follow HTTP redirects.
|
Whether to follow HTTP redirects.
|
||||||
@@ -97,15 +99,13 @@ class HTMLScraper(BaseScraper):
|
|||||||
|
|
||||||
base_ct = raw_ct.split(";", 1)[0].strip().lower()
|
base_ct = raw_ct.split(";", 1)[0].strip().lower()
|
||||||
if base_ct != self.content_type.value:
|
if base_ct != self.content_type.value:
|
||||||
raise ValueError(
|
raise ValueError(f"Expected HTML content, got '{raw_ct}'")
|
||||||
f"Expected HTML content, got '{raw_ct}'"
|
|
||||||
)
|
|
||||||
|
|
||||||
def fetch(
|
def fetch(
|
||||||
self,
|
self,
|
||||||
source: str,
|
source: str,
|
||||||
*,
|
*,
|
||||||
metadata: Optional[Mapping[str, Any]] = None,
|
metadata: Mapping[str, Any] | None = None,
|
||||||
) -> Content:
|
) -> Content:
|
||||||
"""
|
"""
|
||||||
Fetch an HTML document from the given source.
|
Fetch an HTML document from the given source.
|
||||||
@@ -113,7 +113,7 @@ class HTMLScraper(BaseScraper):
|
|||||||
Args:
|
Args:
|
||||||
source (str):
|
source (str):
|
||||||
URL of the HTML document.
|
URL of the HTML document.
|
||||||
metadata (Optional[Mapping[str, Any]], optional):
|
metadata (Mapping[str, Any] | None, optional):
|
||||||
Optional metadata to be merged into the returned content.
|
Optional metadata to be merged into the returned content.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
|
|||||||
@@ -1,10 +1,22 @@
|
|||||||
|
from collections.abc import Mapping
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from typing import Any, Mapping, Optional
|
|
||||||
from omniread.core.content import Content, ContentType
|
from omniread.core.content import Content, ContentType
|
||||||
from omniread.core.scraper import BaseScraper
|
from omniread.core.scraper import BaseScraper
|
||||||
|
|
||||||
class HTMLScraper(BaseScraper):
|
class HTMLScraper(BaseScraper):
|
||||||
content_type: ContentType
|
content_type: ContentType
|
||||||
def __init__(self, *, client: Optional[httpx.Client] = ..., timeout: float = ..., headers: Optional[Mapping[str, str]] = ..., follow_redirects: bool = ...) -> None: ...
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
client: httpx.Client | None = ...,
|
||||||
|
timeout: float = ...,
|
||||||
|
headers: Mapping[str, str] | None = ...,
|
||||||
|
follow_redirects: bool = ...,
|
||||||
|
) -> None: ...
|
||||||
def validate_content_type(self, response: httpx.Response) -> None: ...
|
def validate_content_type(self, response: httpx.Response) -> None: ...
|
||||||
def fetch(self, source: str, *, metadata: Optional[Mapping[str, Any]] = ...) -> Content: ...
|
def fetch(
|
||||||
|
self, source: str, *, metadata: Mapping[str, Any] | None = ...
|
||||||
|
) -> Content: ...
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
from .client import FileSystemPDFClient
|
from .client import FileSystemPDFClient
|
||||||
from .scraper import PDFScraper
|
|
||||||
from .parser import PDFParser
|
from .parser import PDFParser
|
||||||
|
from .scraper import PDFScraper
|
||||||
|
|
||||||
__all__ = ["FileSystemPDFClient", "PDFScraper", "PDFParser"]
|
__all__ = ["FileSystemPDFClient", "PDFScraper", "PDFParser"]
|
||||||
|
|||||||
@@ -17,9 +17,9 @@ Typical backing stores include:
|
|||||||
- Network file systems
|
- Network file systems
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Any
|
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
class BasePDFClient(ABC):
|
class BasePDFClient(ABC):
|
||||||
|
|||||||
@@ -10,8 +10,8 @@ PDF parsers are responsible for interpreting binary PDF data and producing
|
|||||||
structured representations suitable for downstream consumption.
|
structured representations suitable for downstream consumption.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Generic, TypeVar
|
|
||||||
from abc import abstractmethod
|
from abc import abstractmethod
|
||||||
|
from typing import Generic, TypeVar
|
||||||
|
|
||||||
from omniread.core.content import ContentType
|
from omniread.core.content import ContentType
|
||||||
from omniread.core.parser import BaseParser
|
from omniread.core.parser import BaseParser
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user