11 Commits

Author SHA1 Message Date
8c707840f1 ci(drone): adopt standard PyPI pipeline with tag==version guard
Replace .drone.yml with the canonical Type A template: the tag build now fails unless the git tag equals the pyproject.toml version (both in the publish pipeline and the backfill replay), preventing a tag from silently publishing nothing. Also drops the emoji log decorations to match the standard.
2026-10-15 18:50:44 +05:30
99f3e9d044 chore(drone): precise version check and unified pydoclint gate 2026-10-15 00:09:59 +05:30
b984dd5f42 docs: add wiki, complete lib nav, and rebuild mcp artifacts
- Add hand-written wiki (index, overview, how-to, extending, dev) with
  MkDocs config following the platform anatomy
- Complete the library nav by registering the csv and xlsx groups in
  docforge.nav.yml and docs/mkdocs.lib.yml
- Regenerate lib/MCP outputs with the rebuilt nav
2026-09-16 19:58:59 +05:30
0eaa6ee99a fix: flatten docs/lib layout, remove stale nested docs/lib/omniread
- Remove 21 stale files from docs/lib/omniread/ (core, csv, html, pdf,
  xlsx subdirs + index.md)
- Update docs/lib/index.md to remove stale omniread/ child link
- Regenerate README.md from root docstring (add 3 missing Xlsx entries:
  FileSystemXlsxClient, XlsxScraper, XlsxParser)
- Regenerate via doc-forge build --mkdocs --module omniread --module-is-source
2026-09-15 18:52:01 +05:30
a4d4874b39 chore: standardise .gitignore 2026-09-13 21:05:16 +05:30
22a0524823 chore: drop root mkdocs.yml; adopt per-kind docs/mkdocs.lib.yml as source of truth 2026-09-13 18:23:35 +05:30
cd06de5435 fix: modernize docstring types (List → list, Optional → X | None) to match signatures; regenerate lib/MCP docs (pydoclint zero) 2026-09-13 16:21:09 +05:30
14d89f5f91 feat: add type stubs for csv and xlsx subpackages 2026-09-12 15:26:30 +05:30
7283a3eafd docs: nest lib docs under docs/lib with lib-prefixed nav 2026-09-11 05:38:06 +05:30
3ebfa34733 docs: nest generated docs under docs/lib and docs/mcp 2026-09-10 21:15:47 +05:30
1e7e7c8a6c standardize packaging, tooling, docs, CI, and licensing 2026-09-10 18:49:06 +05:30
106 changed files with 6155 additions and 2715 deletions

View File

@@ -1,3 +1,21 @@
# Drone standard — Type A: PyPI package
#
# Canonical pipeline for internal Python libraries that ship a wheel/sdist to
# the private index (dagpipe, doc-forge, hexa, mail-intake, mongo-ops,
# omniread, openapi-first, py-jwt).
#
# Two pipelines ship together:
# build-and-publish-pypi — tag-triggered build + quality gate + publish.
# backfill-pypi-from-tags — custom-triggered replay of every git tag that is
# missing from the index (disaster recovery).
#
# Secrets (Drone): PIP_REPO_URL, PIP_USERNAME, PIP_PASSWORD.
# Copy to <repo>/.drone.yml unchanged; the package name and version are read
# from pyproject.toml, so no per-repo edits are required.
#
# Version guard: the tag build fails unless the git tag equals the version in
# pyproject.toml (see the versioning standard). This stops a tag from looking
# like a release while silently publishing nothing.
---
kind: pipeline
type: docker
@@ -23,20 +41,46 @@ steps:
commands:
- PACKAGE_NAME=$(grep -E '^name\s*=' pyproject.toml | head -1 | cut -d'"' -f2)
- VERSION=$(grep -E '^version\s*=' pyproject.toml | head -1 | cut -d'"' -f2)
- echo "🔍 Checking if $PACKAGE_NAME==$VERSION exists on $PIP_REPO_URL ..."
- |
if curl -fsSL -u "$PIP_USERNAME:$PIP_PASSWORD" "$PIP_REPO_URL/simple/$PACKAGE_NAME/" | grep -q "$VERSION"; then
echo "✅ $PACKAGE_NAME==$VERSION already exists — skipping build."
if [ "$DRONE_TAG" != "$VERSION" ]; then
echo "Tag '$DRONE_TAG' does not match pyproject version '$VERSION'; refusing to publish."
exit 1
fi
- echo "Checking if $PACKAGE_NAME==$VERSION exists on $PIP_REPO_URL ..."
- |
if curl -fsSL -u "$PIP_USERNAME:$PIP_PASSWORD" "$PIP_REPO_URL/simple/$PACKAGE_NAME/" | grep -qE "${PACKAGE_NAME}-${VERSION}(-|\.tar\.gz|\.whl)"; then
echo "$PACKAGE_NAME==$VERSION already exists; skipping build."
exit 78
else
echo "🆕 New version detected: $PACKAGE_NAME==$VERSION"
echo "New version detected: $PACKAGE_NAME==$VERSION"
fi
- name: quality-gate
image: python:3.13-slim
environment:
PIP_REPO_URL:
from_secret: PIP_REPO_URL
PIP_USERNAME:
from_secret: PIP_USERNAME
PIP_PASSWORD:
from_secret: PIP_PASSWORD
commands:
- pip install --upgrade pip build
- |
AUTH_URL="https://${PIP_USERNAME}:${PIP_PASSWORD}@$(echo "${PIP_REPO_URL#*://}" | sed 's:/*$::')/simple"
pip install --index-url "$AUTH_URL" --extra-index-url https://pypi.org/simple/ -U ".[dev]" "pydoclint>=0.9.1"
- echo "Running quality gate..."
- python -m black --check .
- python -m ruff check .
- python -m mypy
- pydoclint .
- python -m pytest
- name: build-package
image: python:3.13-slim
commands:
- pip install --upgrade pip build
- echo "📦 Building Python package..."
- echo "Building Python package..."
- python -m build
- ls -l dist
@@ -51,7 +95,7 @@ steps:
from_secret: PIP_PASSWORD
commands:
- pip install --upgrade twine
- echo "🚀 Uploading to private PyPI at $PIP_REPO_URL ..."
- echo "Uploading to private PyPI at $PIP_REPO_URL ..."
- |
twine upload \
--repository-url "$PIP_REPO_URL" \
@@ -98,25 +142,31 @@ steps:
set -e
PACKAGE_NAME=$(grep -E '^name\s*=' pyproject.toml | cut -d'"' -f2)
echo "📦 Package: $PACKAGE_NAME"
echo "Package: $PACKAGE_NAME"
for TAG in $(git tag --sort=version:refname); do
VERSION="$TAG"
echo "🔁 Version: $VERSION"
echo "Version: $VERSION"
if curl -fsSL -u "$PIP_USERNAME:$PIP_PASSWORD" \
"$PIP_REPO_URL/simple/$PACKAGE_NAME/" | grep -q "$VERSION"; then
echo "⏭️ Exists, skipping"
"$PIP_REPO_URL/simple/$PACKAGE_NAME/" | grep -qE "${PACKAGE_NAME}-${VERSION}(-|\.tar\.gz|\.whl)"; then
echo "Exists, skipping"
continue
fi
git checkout --force "$TAG"
echo "🏗️ Building $VERSION"
TAG_VERSION=$(grep -E '^version\s*=' pyproject.toml | head -1 | cut -d'"' -f2)
if [ "$TAG" != "$TAG_VERSION" ]; then
echo "Tag $TAG != pyproject version $TAG_VERSION at that commit; skipping"
continue
fi
echo "Building $VERSION"
rm -rf dist
python -m build
echo "⬆️ Uploading $VERSION"
echo "Uploading $VERSION"
twine upload \
--repository-url "$PIP_REPO_URL" \
-u "$PIP_USERNAME" \

10
.gitignore vendored
View File

@@ -25,10 +25,17 @@ env/
htmlcov/
coverage.xml
# Lint / type checking
.mypy_cache/
.ruff_cache/
# Environment variables
.env
.env.local
# Docs build artifacts
site
# OS
.DS_Store
Thumbs.db
@@ -37,5 +44,4 @@ Thumbs.db
*.swp
*.swo
*~
*.tmp
site
*.tmp

21
CHANGELOG.md Normal file
View File

@@ -0,0 +1,21 @@
# Changelog
All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased]
### Added
- `py.typed` marker for PEP 561 type information.
- `.drone.yml` CI with a quality-gate step (black, ruff, mypy, pytest).
- Canonical `docforge.nav.yml`, generated `mkdocs.yml` and `mcp_docs/` via doc-forge.
- MIT `LICENSE`.
### Changed
- Standardized `pyproject.toml` (canonical packaging, lint tool config, extras).
### Fixed
- `XLSX`/`CSV` members added to the `content` type surface.
- Stub fixes for typed API surfaces.

21
LICENSE Normal file
View File

@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2026 Aetoskia Platform
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@@ -91,6 +91,9 @@ required.
- `FileSystemPDFClient`: Local filesystem PDF access.
- `PDFScraper`: PDF-specific content acquisition.
- `PDFParser`: Base parser for PDF binary interpretation.
- `FileSystemXlsxClient`: Local filesystem spreadsheet access.
- `XlsxScraper`: XLSX-specific content acquisition.
- `XlsxParser`: Generic string-row parser for xlsx workbooks.
---

View File

@@ -1,16 +1,31 @@
home: index.md
home: lib/index.md
groups:
Core API:
- core/index.md
- core/content.md
- core/parser.md
- core/scraper.md
- lib/core/index.md
- lib/core/content.md
- lib/core/parser.md
- lib/core/scraper.md
HTML Handling:
- html/index.md
- html/parser.md
- html/scraper.md
- lib/html/index.md
- lib/html/parser.md
- lib/html/scraper.md
PDF Handling:
- pdf/index.md
- pdf/client.md
- pdf/parser.md
- pdf/scraper.md
- lib/pdf/index.md
- lib/pdf/client.md
- lib/pdf/parser.md
- lib/pdf/scraper.md
CSV Handling:
- lib/csv/index.md
- lib/csv/client.md
- lib/csv/parser_base.md
- lib/csv/parser.md
- lib/csv/scraper.md
XLSX Handling:
- lib/xlsx/index.md
- lib/xlsx/client.md
- lib/xlsx/parser_base.md
- lib/xlsx/parser.md
- lib/xlsx/scraper.md
icon:
logo: material/book-open-page-variant
repo: fontawesome/brands/github

3
docs/lib/csv/client.md Normal file
View File

@@ -0,0 +1,3 @@
# Client
::: omniread.csv.client

3
docs/lib/csv/index.md Normal file
View File

@@ -0,0 +1,3 @@
# Csv
::: omniread.csv

3
docs/lib/csv/parser.md Normal file
View File

@@ -0,0 +1,3 @@
# Parser
::: omniread.csv.parser

View File

@@ -0,0 +1,3 @@
# Parser Base
::: omniread.csv.parser_base

3
docs/lib/csv/scraper.md Normal file
View File

@@ -0,0 +1,3 @@
# Scraper
::: omniread.csv.scraper

3
docs/lib/xlsx/client.md Normal file
View File

@@ -0,0 +1,3 @@
# Client
::: omniread.xlsx.client

3
docs/lib/xlsx/index.md Normal file
View File

@@ -0,0 +1,3 @@
# Xlsx
::: omniread.xlsx

3
docs/lib/xlsx/parser.md Normal file
View File

@@ -0,0 +1,3 @@
# Parser
::: omniread.xlsx.parser

View File

@@ -0,0 +1,3 @@
# Parser Base
::: omniread.xlsx.parser_base

3
docs/lib/xlsx/scraper.md Normal file
View File

@@ -0,0 +1,3 @@
# Scraper
::: omniread.xlsx.scraper

View File

@@ -1,6 +1,6 @@
{
"project": "omniread",
"project": "OmniRead",
"type": "docforge-model",
"modules_count": 12,
"modules_count": 22,
"source": "docforge"
}

View File

@@ -4,46 +4,11 @@
"path": "omniread.core.content",
"docstring": "# Summary\n\nCanonical content models for OmniRead.\n\nThis module defines the **format-agnostic content representation** used across\nall parsers and scrapers in OmniRead.\n\nThe models defined here represent *what* was extracted, not *how* it was\nretrieved or parsed. Format-specific behavior and metadata must not alter\nthe semantic meaning of these models.",
"objects": {
"Enum": {
"name": "Enum",
"kind": "alias",
"path": "omniread.core.content.Enum",
"signature": "<bound method Alias.signature of Alias('Enum', 'enum.Enum')>",
"docstring": null
},
"dataclass": {
"name": "dataclass",
"kind": "alias",
"path": "omniread.core.content.dataclass",
"signature": "<bound method Alias.signature of Alias('dataclass', 'dataclasses.dataclass')>",
"docstring": null
},
"Any": {
"name": "Any",
"kind": "alias",
"path": "omniread.core.content.Any",
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
"docstring": null
},
"Mapping": {
"name": "Mapping",
"kind": "alias",
"path": "omniread.core.content.Mapping",
"signature": "<bound method Alias.signature of Alias('Mapping', 'typing.Mapping')>",
"docstring": null
},
"Optional": {
"name": "Optional",
"kind": "alias",
"path": "omniread.core.content.Optional",
"signature": "<bound method Alias.signature of Alias('Optional', 'typing.Optional')>",
"docstring": null
},
"ContentType": {
"name": "ContentType",
"kind": "class",
"path": "omniread.core.content.ContentType",
"signature": "<bound method Class.signature of Class('ContentType', 19, 42)>",
"signature": null,
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
"members": {
"HTML": {
@@ -60,6 +25,20 @@
"signature": null,
"docstring": "PDF document content."
},
"XLSX": {
"name": "XLSX",
"kind": "attribute",
"path": "omniread.core.content.ContentType.XLSX",
"signature": null,
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
},
"CSV": {
"name": "CSV",
"kind": "attribute",
"path": "omniread.core.content.ContentType.CSV",
"signature": null,
"docstring": "Comma-separated-value document content."
},
"JSON": {
"name": "JSON",
"kind": "attribute",
@@ -80,7 +59,7 @@
"name": "Content",
"kind": "class",
"path": "omniread.core.content.Content",
"signature": "<bound method Class.signature of Class('Content', 45, 77)>",
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
"members": {
"raw": {

View File

@@ -8,35 +8,35 @@
"name": "Content",
"kind": "class",
"path": "omniread.core.Content",
"signature": "<bound method Alias.signature of Alias('Content', 'omniread.core.content.Content')>",
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
"members": {
"raw": {
"name": "raw",
"kind": "attribute",
"path": "omniread.core.Content.raw",
"signature": "<bound method Alias.signature of Alias('raw', 'omniread.core.content.Content.raw')>",
"signature": null,
"docstring": "Raw content bytes as retrieved from the source."
},
"source": {
"name": "source",
"kind": "attribute",
"path": "omniread.core.Content.source",
"signature": "<bound method Alias.signature of Alias('source', 'omniread.core.content.Content.source')>",
"signature": null,
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
},
"content_type": {
"name": "content_type",
"kind": "attribute",
"path": "omniread.core.Content.content_type",
"signature": "<bound method Alias.signature of Alias('content_type', 'omniread.core.content.Content.content_type')>",
"signature": null,
"docstring": "Optional MIME type of the content, if known."
},
"metadata": {
"name": "metadata",
"kind": "attribute",
"path": "omniread.core.Content.metadata",
"signature": "<bound method Alias.signature of Alias('metadata', 'omniread.core.content.Content.metadata')>",
"signature": null,
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
}
}
@@ -45,35 +45,49 @@
"name": "ContentType",
"kind": "class",
"path": "omniread.core.ContentType",
"signature": "<bound method Alias.signature of Alias('ContentType', 'omniread.core.content.ContentType')>",
"signature": null,
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
"members": {
"HTML": {
"name": "HTML",
"kind": "attribute",
"path": "omniread.core.ContentType.HTML",
"signature": "<bound method Alias.signature of Alias('HTML', 'omniread.core.content.ContentType.HTML')>",
"signature": null,
"docstring": "HTML document content."
},
"PDF": {
"name": "PDF",
"kind": "attribute",
"path": "omniread.core.ContentType.PDF",
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
"signature": null,
"docstring": "PDF document content."
},
"XLSX": {
"name": "XLSX",
"kind": "attribute",
"path": "omniread.core.ContentType.XLSX",
"signature": null,
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
},
"CSV": {
"name": "CSV",
"kind": "attribute",
"path": "omniread.core.ContentType.CSV",
"signature": null,
"docstring": "Comma-separated-value document content."
},
"JSON": {
"name": "JSON",
"kind": "attribute",
"path": "omniread.core.ContentType.JSON",
"signature": "<bound method Alias.signature of Alias('JSON', 'omniread.core.content.ContentType.JSON')>",
"signature": null,
"docstring": "JSON document content."
},
"XML": {
"name": "XML",
"kind": "attribute",
"path": "omniread.core.ContentType.XML",
"signature": "<bound method Alias.signature of Alias('XML', 'omniread.core.content.ContentType.XML')>",
"signature": null,
"docstring": "XML document content."
}
}
@@ -82,35 +96,35 @@
"name": "BaseParser",
"kind": "class",
"path": "omniread.core.BaseParser",
"signature": "<bound method Alias.signature of Alias('BaseParser', 'omniread.core.parser.BaseParser')>",
"signature": "BaseParser(content: Content)",
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
"members": {
"supported_types": {
"name": "supported_types",
"kind": "attribute",
"path": "omniread.core.BaseParser.supported_types",
"signature": "<bound method Alias.signature of Alias('supported_types', 'omniread.core.parser.BaseParser.supported_types')>",
"signature": null,
"docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic."
},
"content": {
"name": "content",
"kind": "attribute",
"path": "omniread.core.BaseParser.content",
"signature": "<bound method Alias.signature of Alias('content', 'omniread.core.parser.BaseParser.content')>",
"signature": null,
"docstring": null
},
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.core.BaseParser.parse",
"signature": "<bound method Alias.signature of Alias('parse', 'omniread.core.parser.BaseParser.parse')>",
"signature": "parse()",
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
},
"supports": {
"name": "supports",
"kind": "function",
"path": "omniread.core.BaseParser.supports",
"signature": "<bound method Alias.signature of Alias('supports', 'omniread.core.parser.BaseParser.supports')>",
"signature": "supports()",
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
}
}
@@ -119,15 +133,15 @@
"name": "BaseScraper",
"kind": "class",
"path": "omniread.core.BaseScraper",
"signature": "<bound method Alias.signature of Alias('BaseScraper', 'omniread.core.scraper.BaseScraper')>",
"signature": null,
"docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.core.BaseScraper.fetch",
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.core.scraper.BaseScraper.fetch')>",
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
"signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None)",
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
}
}
},
@@ -138,46 +152,11 @@
"signature": null,
"docstring": "# Summary\n\nCanonical content models for OmniRead.\n\nThis module defines the **format-agnostic content representation** used across\nall parsers and scrapers in OmniRead.\n\nThe models defined here represent *what* was extracted, not *how* it was\nretrieved or parsed. Format-specific behavior and metadata must not alter\nthe semantic meaning of these models.",
"members": {
"Enum": {
"name": "Enum",
"kind": "alias",
"path": "omniread.core.content.Enum",
"signature": "<bound method Alias.signature of Alias('Enum', 'enum.Enum')>",
"docstring": null
},
"dataclass": {
"name": "dataclass",
"kind": "alias",
"path": "omniread.core.content.dataclass",
"signature": "<bound method Alias.signature of Alias('dataclass', 'dataclasses.dataclass')>",
"docstring": null
},
"Any": {
"name": "Any",
"kind": "alias",
"path": "omniread.core.content.Any",
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
"docstring": null
},
"Mapping": {
"name": "Mapping",
"kind": "alias",
"path": "omniread.core.content.Mapping",
"signature": "<bound method Alias.signature of Alias('Mapping', 'typing.Mapping')>",
"docstring": null
},
"Optional": {
"name": "Optional",
"kind": "alias",
"path": "omniread.core.content.Optional",
"signature": "<bound method Alias.signature of Alias('Optional', 'typing.Optional')>",
"docstring": null
},
"ContentType": {
"name": "ContentType",
"kind": "class",
"path": "omniread.core.content.ContentType",
"signature": "<bound method Class.signature of Class('ContentType', 19, 42)>",
"signature": null,
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
"members": {
"HTML": {
@@ -194,6 +173,20 @@
"signature": null,
"docstring": "PDF document content."
},
"XLSX": {
"name": "XLSX",
"kind": "attribute",
"path": "omniread.core.content.ContentType.XLSX",
"signature": null,
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
},
"CSV": {
"name": "CSV",
"kind": "attribute",
"path": "omniread.core.content.ContentType.CSV",
"signature": null,
"docstring": "Comma-separated-value document content."
},
"JSON": {
"name": "JSON",
"kind": "attribute",
@@ -214,7 +207,7 @@
"name": "Content",
"kind": "class",
"path": "omniread.core.content.Content",
"signature": "<bound method Class.signature of Class('Content', 45, 77)>",
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
"members": {
"raw": {
@@ -256,74 +249,39 @@
"signature": null,
"docstring": "# Summary\n\nAbstract parsing contracts for OmniRead.\n\nThis module defines the **format-agnostic parser interface** used to transform\nraw content into structured, typed representations.\n\nParsers are responsible for:\n\n- Interpreting a single `Content` instance\n- Validating compatibility with the content type\n- Producing a structured output suitable for downstream consumers\n\nParsers are not responsible for:\n\n- Fetching or acquiring content\n- Performing retries or error recovery\n- Managing multiple content sources",
"members": {
"ABC": {
"name": "ABC",
"kind": "alias",
"path": "omniread.core.parser.ABC",
"signature": "<bound method Alias.signature of Alias('ABC', 'abc.ABC')>",
"docstring": null
},
"abstractmethod": {
"name": "abstractmethod",
"kind": "alias",
"path": "omniread.core.parser.abstractmethod",
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
"docstring": null
},
"Generic": {
"name": "Generic",
"kind": "alias",
"path": "omniread.core.parser.Generic",
"signature": "<bound method Alias.signature of Alias('Generic', 'typing.Generic')>",
"docstring": null
},
"TypeVar": {
"name": "TypeVar",
"kind": "alias",
"path": "omniread.core.parser.TypeVar",
"signature": "<bound method Alias.signature of Alias('TypeVar', 'typing.TypeVar')>",
"docstring": null
},
"Set": {
"name": "Set",
"kind": "alias",
"path": "omniread.core.parser.Set",
"signature": "<bound method Alias.signature of Alias('Set', 'typing.Set')>",
"docstring": null
},
"Content": {
"name": "Content",
"kind": "class",
"path": "omniread.core.parser.Content",
"signature": "<bound method Alias.signature of Alias('Content', 'omniread.core.content.Content')>",
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
"members": {
"raw": {
"name": "raw",
"kind": "attribute",
"path": "omniread.core.parser.Content.raw",
"signature": "<bound method Alias.signature of Alias('raw', 'omniread.core.content.Content.raw')>",
"signature": null,
"docstring": "Raw content bytes as retrieved from the source."
},
"source": {
"name": "source",
"kind": "attribute",
"path": "omniread.core.parser.Content.source",
"signature": "<bound method Alias.signature of Alias('source', 'omniread.core.content.Content.source')>",
"signature": null,
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
},
"content_type": {
"name": "content_type",
"kind": "attribute",
"path": "omniread.core.parser.Content.content_type",
"signature": "<bound method Alias.signature of Alias('content_type', 'omniread.core.content.Content.content_type')>",
"signature": null,
"docstring": "Optional MIME type of the content, if known."
},
"metadata": {
"name": "metadata",
"kind": "attribute",
"path": "omniread.core.parser.Content.metadata",
"signature": "<bound method Alias.signature of Alias('metadata', 'omniread.core.content.Content.metadata')>",
"signature": null,
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
}
}
@@ -332,35 +290,49 @@
"name": "ContentType",
"kind": "class",
"path": "omniread.core.parser.ContentType",
"signature": "<bound method Alias.signature of Alias('ContentType', 'omniread.core.content.ContentType')>",
"signature": null,
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
"members": {
"HTML": {
"name": "HTML",
"kind": "attribute",
"path": "omniread.core.parser.ContentType.HTML",
"signature": "<bound method Alias.signature of Alias('HTML', 'omniread.core.content.ContentType.HTML')>",
"signature": null,
"docstring": "HTML document content."
},
"PDF": {
"name": "PDF",
"kind": "attribute",
"path": "omniread.core.parser.ContentType.PDF",
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
"signature": null,
"docstring": "PDF document content."
},
"XLSX": {
"name": "XLSX",
"kind": "attribute",
"path": "omniread.core.parser.ContentType.XLSX",
"signature": null,
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
},
"CSV": {
"name": "CSV",
"kind": "attribute",
"path": "omniread.core.parser.ContentType.CSV",
"signature": null,
"docstring": "Comma-separated-value document content."
},
"JSON": {
"name": "JSON",
"kind": "attribute",
"path": "omniread.core.parser.ContentType.JSON",
"signature": "<bound method Alias.signature of Alias('JSON', 'omniread.core.content.ContentType.JSON')>",
"signature": null,
"docstring": "JSON document content."
},
"XML": {
"name": "XML",
"kind": "attribute",
"path": "omniread.core.parser.ContentType.XML",
"signature": "<bound method Alias.signature of Alias('XML', 'omniread.core.content.ContentType.XML')>",
"signature": null,
"docstring": "XML document content."
}
}
@@ -376,7 +348,7 @@
"name": "BaseParser",
"kind": "class",
"path": "omniread.core.parser.BaseParser",
"signature": "<bound method Class.signature of Class('BaseParser', 30, 111)>",
"signature": "BaseParser(content: Content)",
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
"members": {
"supported_types": {
@@ -397,14 +369,14 @@
"name": "parse",
"kind": "function",
"path": "omniread.core.parser.BaseParser.parse",
"signature": "<bound method Function.signature of Function('parse', 75, 94)>",
"signature": "parse() -> T",
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
},
"supports": {
"name": "supports",
"kind": "function",
"path": "omniread.core.parser.BaseParser.supports",
"signature": "<bound method Function.signature of Function('supports', 96, 111)>",
"signature": "supports() -> bool",
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
}
}
@@ -418,74 +390,39 @@
"signature": null,
"docstring": "# Summary\n\nAbstract scraping contracts for OmniRead.\n\nThis module defines the **format-agnostic scraper interface** responsible for\nacquiring raw content from external sources.\n\nScrapers are responsible for:\n\n- Locating and retrieving raw content bytes\n- Attaching minimal contextual metadata\n- Returning normalized `Content` objects\n\nScrapers are explicitly NOT responsible for:\n\n- Parsing or interpreting content\n- Inferring structure or semantics\n- Performing content-type specific processing\n\nAll interpretation must be delegated to parsers.",
"members": {
"ABC": {
"name": "ABC",
"kind": "alias",
"path": "omniread.core.scraper.ABC",
"signature": "<bound method Alias.signature of Alias('ABC', 'abc.ABC')>",
"docstring": null
},
"abstractmethod": {
"name": "abstractmethod",
"kind": "alias",
"path": "omniread.core.scraper.abstractmethod",
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
"docstring": null
},
"Any": {
"name": "Any",
"kind": "alias",
"path": "omniread.core.scraper.Any",
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
"docstring": null
},
"Mapping": {
"name": "Mapping",
"kind": "alias",
"path": "omniread.core.scraper.Mapping",
"signature": "<bound method Alias.signature of Alias('Mapping', 'typing.Mapping')>",
"docstring": null
},
"Optional": {
"name": "Optional",
"kind": "alias",
"path": "omniread.core.scraper.Optional",
"signature": "<bound method Alias.signature of Alias('Optional', 'typing.Optional')>",
"docstring": null
},
"Content": {
"name": "Content",
"kind": "class",
"path": "omniread.core.scraper.Content",
"signature": "<bound method Alias.signature of Alias('Content', 'omniread.core.content.Content')>",
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
"members": {
"raw": {
"name": "raw",
"kind": "attribute",
"path": "omniread.core.scraper.Content.raw",
"signature": "<bound method Alias.signature of Alias('raw', 'omniread.core.content.Content.raw')>",
"signature": null,
"docstring": "Raw content bytes as retrieved from the source."
},
"source": {
"name": "source",
"kind": "attribute",
"path": "omniread.core.scraper.Content.source",
"signature": "<bound method Alias.signature of Alias('source', 'omniread.core.content.Content.source')>",
"signature": null,
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
},
"content_type": {
"name": "content_type",
"kind": "attribute",
"path": "omniread.core.scraper.Content.content_type",
"signature": "<bound method Alias.signature of Alias('content_type', 'omniread.core.content.Content.content_type')>",
"signature": null,
"docstring": "Optional MIME type of the content, if known."
},
"metadata": {
"name": "metadata",
"kind": "attribute",
"path": "omniread.core.scraper.Content.metadata",
"signature": "<bound method Alias.signature of Alias('metadata', 'omniread.core.content.Content.metadata')>",
"signature": null,
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
}
}
@@ -494,15 +431,15 @@
"name": "BaseScraper",
"kind": "class",
"path": "omniread.core.scraper.BaseScraper",
"signature": "<bound method Class.signature of Class('BaseScraper', 30, 82)>",
"signature": null,
"docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.core.scraper.BaseScraper.fetch",
"signature": "<bound method Function.signature of Function('fetch', 51, 82)>",
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
"signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None) -> Content",
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
}
}
}

View File

@@ -4,74 +4,39 @@
"path": "omniread.core.parser",
"docstring": "# Summary\n\nAbstract parsing contracts for OmniRead.\n\nThis module defines the **format-agnostic parser interface** used to transform\nraw content into structured, typed representations.\n\nParsers are responsible for:\n\n- Interpreting a single `Content` instance\n- Validating compatibility with the content type\n- Producing a structured output suitable for downstream consumers\n\nParsers are not responsible for:\n\n- Fetching or acquiring content\n- Performing retries or error recovery\n- Managing multiple content sources",
"objects": {
"ABC": {
"name": "ABC",
"kind": "alias",
"path": "omniread.core.parser.ABC",
"signature": "<bound method Alias.signature of Alias('ABC', 'abc.ABC')>",
"docstring": null
},
"abstractmethod": {
"name": "abstractmethod",
"kind": "alias",
"path": "omniread.core.parser.abstractmethod",
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
"docstring": null
},
"Generic": {
"name": "Generic",
"kind": "alias",
"path": "omniread.core.parser.Generic",
"signature": "<bound method Alias.signature of Alias('Generic', 'typing.Generic')>",
"docstring": null
},
"TypeVar": {
"name": "TypeVar",
"kind": "alias",
"path": "omniread.core.parser.TypeVar",
"signature": "<bound method Alias.signature of Alias('TypeVar', 'typing.TypeVar')>",
"docstring": null
},
"Set": {
"name": "Set",
"kind": "alias",
"path": "omniread.core.parser.Set",
"signature": "<bound method Alias.signature of Alias('Set', 'typing.Set')>",
"docstring": null
},
"Content": {
"name": "Content",
"kind": "class",
"path": "omniread.core.parser.Content",
"signature": "<bound method Alias.signature of Alias('Content', 'omniread.core.content.Content')>",
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
"members": {
"raw": {
"name": "raw",
"kind": "attribute",
"path": "omniread.core.parser.Content.raw",
"signature": "<bound method Alias.signature of Alias('raw', 'omniread.core.content.Content.raw')>",
"signature": null,
"docstring": "Raw content bytes as retrieved from the source."
},
"source": {
"name": "source",
"kind": "attribute",
"path": "omniread.core.parser.Content.source",
"signature": "<bound method Alias.signature of Alias('source', 'omniread.core.content.Content.source')>",
"signature": null,
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
},
"content_type": {
"name": "content_type",
"kind": "attribute",
"path": "omniread.core.parser.Content.content_type",
"signature": "<bound method Alias.signature of Alias('content_type', 'omniread.core.content.Content.content_type')>",
"signature": null,
"docstring": "Optional MIME type of the content, if known."
},
"metadata": {
"name": "metadata",
"kind": "attribute",
"path": "omniread.core.parser.Content.metadata",
"signature": "<bound method Alias.signature of Alias('metadata', 'omniread.core.content.Content.metadata')>",
"signature": null,
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
}
}
@@ -80,35 +45,49 @@
"name": "ContentType",
"kind": "class",
"path": "omniread.core.parser.ContentType",
"signature": "<bound method Alias.signature of Alias('ContentType', 'omniread.core.content.ContentType')>",
"signature": null,
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
"members": {
"HTML": {
"name": "HTML",
"kind": "attribute",
"path": "omniread.core.parser.ContentType.HTML",
"signature": "<bound method Alias.signature of Alias('HTML', 'omniread.core.content.ContentType.HTML')>",
"signature": null,
"docstring": "HTML document content."
},
"PDF": {
"name": "PDF",
"kind": "attribute",
"path": "omniread.core.parser.ContentType.PDF",
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
"signature": null,
"docstring": "PDF document content."
},
"XLSX": {
"name": "XLSX",
"kind": "attribute",
"path": "omniread.core.parser.ContentType.XLSX",
"signature": null,
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
},
"CSV": {
"name": "CSV",
"kind": "attribute",
"path": "omniread.core.parser.ContentType.CSV",
"signature": null,
"docstring": "Comma-separated-value document content."
},
"JSON": {
"name": "JSON",
"kind": "attribute",
"path": "omniread.core.parser.ContentType.JSON",
"signature": "<bound method Alias.signature of Alias('JSON', 'omniread.core.content.ContentType.JSON')>",
"signature": null,
"docstring": "JSON document content."
},
"XML": {
"name": "XML",
"kind": "attribute",
"path": "omniread.core.parser.ContentType.XML",
"signature": "<bound method Alias.signature of Alias('XML', 'omniread.core.content.ContentType.XML')>",
"signature": null,
"docstring": "XML document content."
}
}
@@ -124,7 +103,7 @@
"name": "BaseParser",
"kind": "class",
"path": "omniread.core.parser.BaseParser",
"signature": "<bound method Class.signature of Class('BaseParser', 30, 111)>",
"signature": "BaseParser(content: Content)",
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
"members": {
"supported_types": {
@@ -145,14 +124,14 @@
"name": "parse",
"kind": "function",
"path": "omniread.core.parser.BaseParser.parse",
"signature": "<bound method Function.signature of Function('parse', 75, 94)>",
"signature": "parse() -> T",
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
},
"supports": {
"name": "supports",
"kind": "function",
"path": "omniread.core.parser.BaseParser.supports",
"signature": "<bound method Function.signature of Function('supports', 96, 111)>",
"signature": "supports() -> bool",
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
}
}

View File

@@ -4,74 +4,39 @@
"path": "omniread.core.scraper",
"docstring": "# Summary\n\nAbstract scraping contracts for OmniRead.\n\nThis module defines the **format-agnostic scraper interface** responsible for\nacquiring raw content from external sources.\n\nScrapers are responsible for:\n\n- Locating and retrieving raw content bytes\n- Attaching minimal contextual metadata\n- Returning normalized `Content` objects\n\nScrapers are explicitly NOT responsible for:\n\n- Parsing or interpreting content\n- Inferring structure or semantics\n- Performing content-type specific processing\n\nAll interpretation must be delegated to parsers.",
"objects": {
"ABC": {
"name": "ABC",
"kind": "alias",
"path": "omniread.core.scraper.ABC",
"signature": "<bound method Alias.signature of Alias('ABC', 'abc.ABC')>",
"docstring": null
},
"abstractmethod": {
"name": "abstractmethod",
"kind": "alias",
"path": "omniread.core.scraper.abstractmethod",
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
"docstring": null
},
"Any": {
"name": "Any",
"kind": "alias",
"path": "omniread.core.scraper.Any",
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
"docstring": null
},
"Mapping": {
"name": "Mapping",
"kind": "alias",
"path": "omniread.core.scraper.Mapping",
"signature": "<bound method Alias.signature of Alias('Mapping', 'typing.Mapping')>",
"docstring": null
},
"Optional": {
"name": "Optional",
"kind": "alias",
"path": "omniread.core.scraper.Optional",
"signature": "<bound method Alias.signature of Alias('Optional', 'typing.Optional')>",
"docstring": null
},
"Content": {
"name": "Content",
"kind": "class",
"path": "omniread.core.scraper.Content",
"signature": "<bound method Alias.signature of Alias('Content', 'omniread.core.content.Content')>",
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
"members": {
"raw": {
"name": "raw",
"kind": "attribute",
"path": "omniread.core.scraper.Content.raw",
"signature": "<bound method Alias.signature of Alias('raw', 'omniread.core.content.Content.raw')>",
"signature": null,
"docstring": "Raw content bytes as retrieved from the source."
},
"source": {
"name": "source",
"kind": "attribute",
"path": "omniread.core.scraper.Content.source",
"signature": "<bound method Alias.signature of Alias('source', 'omniread.core.content.Content.source')>",
"signature": null,
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
},
"content_type": {
"name": "content_type",
"kind": "attribute",
"path": "omniread.core.scraper.Content.content_type",
"signature": "<bound method Alias.signature of Alias('content_type', 'omniread.core.content.Content.content_type')>",
"signature": null,
"docstring": "Optional MIME type of the content, if known."
},
"metadata": {
"name": "metadata",
"kind": "attribute",
"path": "omniread.core.scraper.Content.metadata",
"signature": "<bound method Alias.signature of Alias('metadata', 'omniread.core.content.Content.metadata')>",
"signature": null,
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
}
}
@@ -80,15 +45,15 @@
"name": "BaseScraper",
"kind": "class",
"path": "omniread.core.scraper.BaseScraper",
"signature": "<bound method Class.signature of Class('BaseScraper', 30, 82)>",
"signature": null,
"docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.core.scraper.BaseScraper.fetch",
"signature": "<bound method Function.signature of Function('fetch', 51, 82)>",
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
"signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None) -> Content",
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
}
}
}

View File

@@ -0,0 +1,41 @@
{
"module": "omniread.csv.client",
"content": {
"path": "omniread.csv.client",
"docstring": "# Summary\n\nCSV client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw\ncomma-separated-value document bytes from a concrete backing store.\n\nClients provide low-level access to csv binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems",
"objects": {
"BaseCsvClient": {
"name": "BaseCsvClient",
"kind": "class",
"path": "omniread.csv.client.BaseCsvClient",
"signature": null,
"docstring": "Abstract client responsible for retrieving csv bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full csv binary payload.\n - Raise retrieval-specific errors on failure.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.csv.client.BaseCsvClient.fetch",
"signature": "fetch(source: Any) -> bytes",
"docstring": "Fetch raw csv bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
}
}
},
"FileSystemCsvClient": {
"name": "FileSystemCsvClient",
"kind": "class",
"path": "omniread.csv.client.FileSystemCsvClient",
"signature": null,
"docstring": "CSV client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads csv files directly from the disk and\n returns their raw binary contents.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.csv.client.FileSystemCsvClient.fetch",
"signature": "fetch(path: Path) -> bytes",
"docstring": "Read a csv file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the csv file.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file."
}
}
}
}
}
}

View File

@@ -0,0 +1,492 @@
{
"module": "omniread.csv",
"content": {
"path": "omniread.csv",
"docstring": "# Summary\n\nCSV subpackage for OmniRead.\n\nProvides acquisition and parsing of comma-separated-value content:\n\n- `BaseCsvClient`: abstract backing-store client for csv bytes.\n- `FileSystemCsvClient`: local filesystem implementation.\n- `CsvScraper`: wraps fetched bytes into canonical `Content`.\n- `CsvParserBase`: content-type-enforcing parser contract.\n- `CsvParser`: generic string-row parser built on the standard csv module.",
"objects": {
"BaseCsvClient": {
"name": "BaseCsvClient",
"kind": "class",
"path": "omniread.csv.BaseCsvClient",
"signature": null,
"docstring": "Abstract client responsible for retrieving csv bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full csv binary payload.\n - Raise retrieval-specific errors on failure.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.csv.BaseCsvClient.fetch",
"signature": "fetch(source: Any)",
"docstring": "Fetch raw csv bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
}
}
},
"FileSystemCsvClient": {
"name": "FileSystemCsvClient",
"kind": "class",
"path": "omniread.csv.FileSystemCsvClient",
"signature": null,
"docstring": "CSV client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads csv files directly from the disk and\n returns their raw binary contents.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.csv.FileSystemCsvClient.fetch",
"signature": "fetch(path: Path)",
"docstring": "Read a csv file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the csv file.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file."
}
}
},
"CsvParser": {
"name": "CsvParser",
"kind": "class",
"path": "omniread.csv.CsvParser",
"signature": "CsvParser(content: Content)",
"docstring": "Generic csv parser producing string rows from the document.\n\nNotes:\n **Responsibilities:**\n\n - Decode the payload (UTF-8 with BOM support, Latin-1 fallback).\n - Detect the delimiter from a leading sample (`,` `;` tab `|`),\n defaulting to `,`.\n - Normalize cells into deterministic stripped string values.\n - Expose row extraction helpers mirroring `XlsxParser.rows`.\n\n **Constraints:**\n\n - All values are strings; consumers requiring typed values must\n convert on their side.\n - Quoted fields containing delimiters/newlines are handled by\n the standard ``csv`` module.",
"members": {
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.csv.CsvParser.parse",
"signature": "parse()",
"docstring": "Parse the document into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the document."
},
"rows": {
"name": "rows",
"kind": "function",
"path": "omniread.csv.CsvParser.rows",
"signature": "rows(*, skip_empty: bool = True)",
"docstring": "Extract normalized string rows from the document.\n\nArgs:\n skip_empty (bool):\n When True (default), rows whose cells are all blank are\n omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row."
}
}
},
"CsvParserBase": {
"name": "CsvParserBase",
"kind": "class",
"path": "omniread.csv.CsvParserBase",
"signature": null,
"docstring": "Base csv parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces csv content-type compatibility and provides\n the extension point for implementing concrete csv parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
"members": {
"supported_types": {
"name": "supported_types",
"kind": "attribute",
"path": "omniread.csv.CsvParserBase.supported_types",
"signature": null,
"docstring": "Set of content types supported by this parser (CSV only)."
},
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.csv.CsvParserBase.parse",
"signature": "parse()",
"docstring": "Parse csv content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
}
}
},
"CsvScraper": {
"name": "CsvScraper",
"kind": "class",
"path": "omniread.csv.CsvScraper",
"signature": "CsvScraper(*, client: BaseCsvClient)",
"docstring": "Scraper for csv documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw csv bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n CSV content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.csv.CsvScraper.fetch",
"signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None)",
"docstring": "Fetch a csv document from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw csv bytes, source\n identifier, CSV content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client."
}
}
},
"client": {
"name": "client",
"kind": "module",
"path": "omniread.csv.client",
"signature": null,
"docstring": "# Summary\n\nCSV client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw\ncomma-separated-value document bytes from a concrete backing store.\n\nClients provide low-level access to csv binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems",
"members": {
"BaseCsvClient": {
"name": "BaseCsvClient",
"kind": "class",
"path": "omniread.csv.client.BaseCsvClient",
"signature": null,
"docstring": "Abstract client responsible for retrieving csv bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full csv binary payload.\n - Raise retrieval-specific errors on failure.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.csv.client.BaseCsvClient.fetch",
"signature": "fetch(source: Any) -> bytes",
"docstring": "Fetch raw csv bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
}
}
},
"FileSystemCsvClient": {
"name": "FileSystemCsvClient",
"kind": "class",
"path": "omniread.csv.client.FileSystemCsvClient",
"signature": null,
"docstring": "CSV client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads csv files directly from the disk and\n returns their raw binary contents.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.csv.client.FileSystemCsvClient.fetch",
"signature": "fetch(path: Path) -> bytes",
"docstring": "Read a csv file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the csv file.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file."
}
}
}
}
},
"parser": {
"name": "parser",
"kind": "module",
"path": "omniread.csv.parser",
"signature": null,
"docstring": "# Summary\n\nCSV parser implementations for OmniRead.\n\nThis module provides a concrete, generic parser for comma-separated-value\ndocuments. It exposes records as lists of string cells so downstream\nconsumers can interpret tabular content without depending on the ``csv``\nmodule directly.\n\nThe parser is intentionally statement-agnostic: it performs no header\ndetection or column interpretation beyond basic cell normalization and\ndelimiter detection.",
"members": {
"Content": {
"name": "Content",
"kind": "class",
"path": "omniread.csv.parser.Content",
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
"members": {
"raw": {
"name": "raw",
"kind": "attribute",
"path": "omniread.csv.parser.Content.raw",
"signature": null,
"docstring": "Raw content bytes as retrieved from the source."
},
"source": {
"name": "source",
"kind": "attribute",
"path": "omniread.csv.parser.Content.source",
"signature": null,
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
},
"content_type": {
"name": "content_type",
"kind": "attribute",
"path": "omniread.csv.parser.Content.content_type",
"signature": null,
"docstring": "Optional MIME type of the content, if known."
},
"metadata": {
"name": "metadata",
"kind": "attribute",
"path": "omniread.csv.parser.Content.metadata",
"signature": null,
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
}
}
},
"CsvParserBase": {
"name": "CsvParserBase",
"kind": "class",
"path": "omniread.csv.parser.CsvParserBase",
"signature": null,
"docstring": "Base csv parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces csv content-type compatibility and provides\n the extension point for implementing concrete csv parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
"members": {
"supported_types": {
"name": "supported_types",
"kind": "attribute",
"path": "omniread.csv.parser.CsvParserBase.supported_types",
"signature": null,
"docstring": "Set of content types supported by this parser (CSV only)."
},
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.csv.parser.CsvParserBase.parse",
"signature": "parse()",
"docstring": "Parse csv content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
}
}
},
"CsvParser": {
"name": "CsvParser",
"kind": "class",
"path": "omniread.csv.parser.CsvParser",
"signature": "CsvParser(content: Content)",
"docstring": "Generic csv parser producing string rows from the document.\n\nNotes:\n **Responsibilities:**\n\n - Decode the payload (UTF-8 with BOM support, Latin-1 fallback).\n - Detect the delimiter from a leading sample (`,` `;` tab `|`),\n defaulting to `,`.\n - Normalize cells into deterministic stripped string values.\n - Expose row extraction helpers mirroring `XlsxParser.rows`.\n\n **Constraints:**\n\n - All values are strings; consumers requiring typed values must\n convert on their side.\n - Quoted fields containing delimiters/newlines are handled by\n the standard ``csv`` module.",
"members": {
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.csv.parser.CsvParser.parse",
"signature": "parse() -> list[list[str]]",
"docstring": "Parse the document into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the document."
},
"rows": {
"name": "rows",
"kind": "function",
"path": "omniread.csv.parser.CsvParser.rows",
"signature": "rows(*, skip_empty: bool = True) -> list[list[str]]",
"docstring": "Extract normalized string rows from the document.\n\nArgs:\n skip_empty (bool):\n When True (default), rows whose cells are all blank are\n omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row."
}
}
}
}
},
"parser_base": {
"name": "parser_base",
"kind": "module",
"path": "omniread.csv.parser_base",
"signature": null,
"docstring": "# Summary\n\nCSV parser base implementation for OmniRead.\n\nThis module defines the **CSV-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for\ncomma-separated-value documents.",
"members": {
"ContentType": {
"name": "ContentType",
"kind": "class",
"path": "omniread.csv.parser_base.ContentType",
"signature": null,
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
"members": {
"HTML": {
"name": "HTML",
"kind": "attribute",
"path": "omniread.csv.parser_base.ContentType.HTML",
"signature": null,
"docstring": "HTML document content."
},
"PDF": {
"name": "PDF",
"kind": "attribute",
"path": "omniread.csv.parser_base.ContentType.PDF",
"signature": null,
"docstring": "PDF document content."
},
"XLSX": {
"name": "XLSX",
"kind": "attribute",
"path": "omniread.csv.parser_base.ContentType.XLSX",
"signature": null,
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
},
"CSV": {
"name": "CSV",
"kind": "attribute",
"path": "omniread.csv.parser_base.ContentType.CSV",
"signature": null,
"docstring": "Comma-separated-value document content."
},
"JSON": {
"name": "JSON",
"kind": "attribute",
"path": "omniread.csv.parser_base.ContentType.JSON",
"signature": null,
"docstring": "JSON document content."
},
"XML": {
"name": "XML",
"kind": "attribute",
"path": "omniread.csv.parser_base.ContentType.XML",
"signature": null,
"docstring": "XML document content."
}
}
},
"BaseParser": {
"name": "BaseParser",
"kind": "class",
"path": "omniread.csv.parser_base.BaseParser",
"signature": "BaseParser(content: Content)",
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
"members": {
"supported_types": {
"name": "supported_types",
"kind": "attribute",
"path": "omniread.csv.parser_base.BaseParser.supported_types",
"signature": null,
"docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic."
},
"content": {
"name": "content",
"kind": "attribute",
"path": "omniread.csv.parser_base.BaseParser.content",
"signature": null,
"docstring": null
},
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.csv.parser_base.BaseParser.parse",
"signature": "parse()",
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
},
"supports": {
"name": "supports",
"kind": "function",
"path": "omniread.csv.parser_base.BaseParser.supports",
"signature": "supports()",
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
}
}
},
"T": {
"name": "T",
"kind": "attribute",
"path": "omniread.csv.parser_base.T",
"signature": null,
"docstring": null
},
"CsvParserBase": {
"name": "CsvParserBase",
"kind": "class",
"path": "omniread.csv.parser_base.CsvParserBase",
"signature": null,
"docstring": "Base csv parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces csv content-type compatibility and provides\n the extension point for implementing concrete csv parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
"members": {
"supported_types": {
"name": "supported_types",
"kind": "attribute",
"path": "omniread.csv.parser_base.CsvParserBase.supported_types",
"signature": null,
"docstring": "Set of content types supported by this parser (CSV only)."
},
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.csv.parser_base.CsvParserBase.parse",
"signature": "parse() -> T",
"docstring": "Parse csv content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
}
}
}
}
},
"scraper": {
"name": "scraper",
"kind": "module",
"path": "omniread.csv.scraper",
"signature": null,
"docstring": "# Summary\n\nCSV scraper for OmniRead.\n\nThis module defines the scraper responsible for acquiring raw\ncomma-separated-value document content from a backing store via a\nconfigured client.\n\nThe scraper does not interpret or parse the acquired bytes; it wraps them in\nthe canonical `Content` model.",
"members": {
"Content": {
"name": "Content",
"kind": "class",
"path": "omniread.csv.scraper.Content",
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
"members": {
"raw": {
"name": "raw",
"kind": "attribute",
"path": "omniread.csv.scraper.Content.raw",
"signature": null,
"docstring": "Raw content bytes as retrieved from the source."
},
"source": {
"name": "source",
"kind": "attribute",
"path": "omniread.csv.scraper.Content.source",
"signature": null,
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
},
"content_type": {
"name": "content_type",
"kind": "attribute",
"path": "omniread.csv.scraper.Content.content_type",
"signature": null,
"docstring": "Optional MIME type of the content, if known."
},
"metadata": {
"name": "metadata",
"kind": "attribute",
"path": "omniread.csv.scraper.Content.metadata",
"signature": null,
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
}
}
},
"ContentType": {
"name": "ContentType",
"kind": "class",
"path": "omniread.csv.scraper.ContentType",
"signature": null,
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
"members": {
"HTML": {
"name": "HTML",
"kind": "attribute",
"path": "omniread.csv.scraper.ContentType.HTML",
"signature": null,
"docstring": "HTML document content."
},
"PDF": {
"name": "PDF",
"kind": "attribute",
"path": "omniread.csv.scraper.ContentType.PDF",
"signature": null,
"docstring": "PDF document content."
},
"XLSX": {
"name": "XLSX",
"kind": "attribute",
"path": "omniread.csv.scraper.ContentType.XLSX",
"signature": null,
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
},
"CSV": {
"name": "CSV",
"kind": "attribute",
"path": "omniread.csv.scraper.ContentType.CSV",
"signature": null,
"docstring": "Comma-separated-value document content."
},
"JSON": {
"name": "JSON",
"kind": "attribute",
"path": "omniread.csv.scraper.ContentType.JSON",
"signature": null,
"docstring": "JSON document content."
},
"XML": {
"name": "XML",
"kind": "attribute",
"path": "omniread.csv.scraper.ContentType.XML",
"signature": null,
"docstring": "XML document content."
}
}
},
"BaseCsvClient": {
"name": "BaseCsvClient",
"kind": "class",
"path": "omniread.csv.scraper.BaseCsvClient",
"signature": null,
"docstring": "Abstract client responsible for retrieving csv bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full csv binary payload.\n - Raise retrieval-specific errors on failure.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.csv.scraper.BaseCsvClient.fetch",
"signature": "fetch(source: Any)",
"docstring": "Fetch raw csv bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
}
}
},
"CsvScraper": {
"name": "CsvScraper",
"kind": "class",
"path": "omniread.csv.scraper.CsvScraper",
"signature": "CsvScraper(*, client: BaseCsvClient)",
"docstring": "Scraper for csv documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw csv bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n CSV content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.csv.scraper.CsvScraper.fetch",
"signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None) -> Content",
"docstring": "Fetch a csv document from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw csv bytes, source\n identifier, CSV content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client."
}
}
}
}
}
}
}
}

View File

@@ -0,0 +1,92 @@
{
"module": "omniread.csv.parser",
"content": {
"path": "omniread.csv.parser",
"docstring": "# Summary\n\nCSV parser implementations for OmniRead.\n\nThis module provides a concrete, generic parser for comma-separated-value\ndocuments. It exposes records as lists of string cells so downstream\nconsumers can interpret tabular content without depending on the ``csv``\nmodule directly.\n\nThe parser is intentionally statement-agnostic: it performs no header\ndetection or column interpretation beyond basic cell normalization and\ndelimiter detection.",
"objects": {
"Content": {
"name": "Content",
"kind": "class",
"path": "omniread.csv.parser.Content",
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
"members": {
"raw": {
"name": "raw",
"kind": "attribute",
"path": "omniread.csv.parser.Content.raw",
"signature": null,
"docstring": "Raw content bytes as retrieved from the source."
},
"source": {
"name": "source",
"kind": "attribute",
"path": "omniread.csv.parser.Content.source",
"signature": null,
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
},
"content_type": {
"name": "content_type",
"kind": "attribute",
"path": "omniread.csv.parser.Content.content_type",
"signature": null,
"docstring": "Optional MIME type of the content, if known."
},
"metadata": {
"name": "metadata",
"kind": "attribute",
"path": "omniread.csv.parser.Content.metadata",
"signature": null,
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
}
}
},
"CsvParserBase": {
"name": "CsvParserBase",
"kind": "class",
"path": "omniread.csv.parser.CsvParserBase",
"signature": null,
"docstring": "Base csv parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces csv content-type compatibility and provides\n the extension point for implementing concrete csv parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
"members": {
"supported_types": {
"name": "supported_types",
"kind": "attribute",
"path": "omniread.csv.parser.CsvParserBase.supported_types",
"signature": null,
"docstring": "Set of content types supported by this parser (CSV only)."
},
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.csv.parser.CsvParserBase.parse",
"signature": "parse()",
"docstring": "Parse csv content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
}
}
},
"CsvParser": {
"name": "CsvParser",
"kind": "class",
"path": "omniread.csv.parser.CsvParser",
"signature": "CsvParser(content: Content)",
"docstring": "Generic csv parser producing string rows from the document.\n\nNotes:\n **Responsibilities:**\n\n - Decode the payload (UTF-8 with BOM support, Latin-1 fallback).\n - Detect the delimiter from a leading sample (`,` `;` tab `|`),\n defaulting to `,`.\n - Normalize cells into deterministic stripped string values.\n - Expose row extraction helpers mirroring `XlsxParser.rows`.\n\n **Constraints:**\n\n - All values are strings; consumers requiring typed values must\n convert on their side.\n - Quoted fields containing delimiters/newlines are handled by\n the standard ``csv`` module.",
"members": {
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.csv.parser.CsvParser.parse",
"signature": "parse() -> list[list[str]]",
"docstring": "Parse the document into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the document."
},
"rows": {
"name": "rows",
"kind": "function",
"path": "omniread.csv.parser.CsvParser.rows",
"signature": "rows(*, skip_empty: bool = True) -> list[list[str]]",
"docstring": "Extract normalized string rows from the document.\n\nArgs:\n skip_empty (bool):\n When True (default), rows whose cells are all blank are\n omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row."
}
}
}
}
}
}

View File

@@ -0,0 +1,127 @@
{
"module": "omniread.csv.parser_base",
"content": {
"path": "omniread.csv.parser_base",
"docstring": "# Summary\n\nCSV parser base implementation for OmniRead.\n\nThis module defines the **CSV-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for\ncomma-separated-value documents.",
"objects": {
"ContentType": {
"name": "ContentType",
"kind": "class",
"path": "omniread.csv.parser_base.ContentType",
"signature": null,
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
"members": {
"HTML": {
"name": "HTML",
"kind": "attribute",
"path": "omniread.csv.parser_base.ContentType.HTML",
"signature": null,
"docstring": "HTML document content."
},
"PDF": {
"name": "PDF",
"kind": "attribute",
"path": "omniread.csv.parser_base.ContentType.PDF",
"signature": null,
"docstring": "PDF document content."
},
"XLSX": {
"name": "XLSX",
"kind": "attribute",
"path": "omniread.csv.parser_base.ContentType.XLSX",
"signature": null,
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
},
"CSV": {
"name": "CSV",
"kind": "attribute",
"path": "omniread.csv.parser_base.ContentType.CSV",
"signature": null,
"docstring": "Comma-separated-value document content."
},
"JSON": {
"name": "JSON",
"kind": "attribute",
"path": "omniread.csv.parser_base.ContentType.JSON",
"signature": null,
"docstring": "JSON document content."
},
"XML": {
"name": "XML",
"kind": "attribute",
"path": "omniread.csv.parser_base.ContentType.XML",
"signature": null,
"docstring": "XML document content."
}
}
},
"BaseParser": {
"name": "BaseParser",
"kind": "class",
"path": "omniread.csv.parser_base.BaseParser",
"signature": "BaseParser(content: Content)",
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
"members": {
"supported_types": {
"name": "supported_types",
"kind": "attribute",
"path": "omniread.csv.parser_base.BaseParser.supported_types",
"signature": null,
"docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic."
},
"content": {
"name": "content",
"kind": "attribute",
"path": "omniread.csv.parser_base.BaseParser.content",
"signature": null,
"docstring": null
},
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.csv.parser_base.BaseParser.parse",
"signature": "parse()",
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
},
"supports": {
"name": "supports",
"kind": "function",
"path": "omniread.csv.parser_base.BaseParser.supports",
"signature": "supports()",
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
}
}
},
"T": {
"name": "T",
"kind": "attribute",
"path": "omniread.csv.parser_base.T",
"signature": null,
"docstring": null
},
"CsvParserBase": {
"name": "CsvParserBase",
"kind": "class",
"path": "omniread.csv.parser_base.CsvParserBase",
"signature": null,
"docstring": "Base csv parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces csv content-type compatibility and provides\n the extension point for implementing concrete csv parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
"members": {
"supported_types": {
"name": "supported_types",
"kind": "attribute",
"path": "omniread.csv.parser_base.CsvParserBase.supported_types",
"signature": null,
"docstring": "Set of content types supported by this parser (CSV only)."
},
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.csv.parser_base.CsvParserBase.parse",
"signature": "parse() -> T",
"docstring": "Parse csv content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
}
}
}
}
}
}

View File

@@ -0,0 +1,129 @@
{
"module": "omniread.csv.scraper",
"content": {
"path": "omniread.csv.scraper",
"docstring": "# Summary\n\nCSV scraper for OmniRead.\n\nThis module defines the scraper responsible for acquiring raw\ncomma-separated-value document content from a backing store via a\nconfigured client.\n\nThe scraper does not interpret or parse the acquired bytes; it wraps them in\nthe canonical `Content` model.",
"objects": {
"Content": {
"name": "Content",
"kind": "class",
"path": "omniread.csv.scraper.Content",
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
"members": {
"raw": {
"name": "raw",
"kind": "attribute",
"path": "omniread.csv.scraper.Content.raw",
"signature": null,
"docstring": "Raw content bytes as retrieved from the source."
},
"source": {
"name": "source",
"kind": "attribute",
"path": "omniread.csv.scraper.Content.source",
"signature": null,
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
},
"content_type": {
"name": "content_type",
"kind": "attribute",
"path": "omniread.csv.scraper.Content.content_type",
"signature": null,
"docstring": "Optional MIME type of the content, if known."
},
"metadata": {
"name": "metadata",
"kind": "attribute",
"path": "omniread.csv.scraper.Content.metadata",
"signature": null,
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
}
}
},
"ContentType": {
"name": "ContentType",
"kind": "class",
"path": "omniread.csv.scraper.ContentType",
"signature": null,
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
"members": {
"HTML": {
"name": "HTML",
"kind": "attribute",
"path": "omniread.csv.scraper.ContentType.HTML",
"signature": null,
"docstring": "HTML document content."
},
"PDF": {
"name": "PDF",
"kind": "attribute",
"path": "omniread.csv.scraper.ContentType.PDF",
"signature": null,
"docstring": "PDF document content."
},
"XLSX": {
"name": "XLSX",
"kind": "attribute",
"path": "omniread.csv.scraper.ContentType.XLSX",
"signature": null,
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
},
"CSV": {
"name": "CSV",
"kind": "attribute",
"path": "omniread.csv.scraper.ContentType.CSV",
"signature": null,
"docstring": "Comma-separated-value document content."
},
"JSON": {
"name": "JSON",
"kind": "attribute",
"path": "omniread.csv.scraper.ContentType.JSON",
"signature": null,
"docstring": "JSON document content."
},
"XML": {
"name": "XML",
"kind": "attribute",
"path": "omniread.csv.scraper.ContentType.XML",
"signature": null,
"docstring": "XML document content."
}
}
},
"BaseCsvClient": {
"name": "BaseCsvClient",
"kind": "class",
"path": "omniread.csv.scraper.BaseCsvClient",
"signature": null,
"docstring": "Abstract client responsible for retrieving csv bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full csv binary payload.\n - Raise retrieval-specific errors on failure.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.csv.scraper.BaseCsvClient.fetch",
"signature": "fetch(source: Any)",
"docstring": "Fetch raw csv bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
}
}
},
"CsvScraper": {
"name": "CsvScraper",
"kind": "class",
"path": "omniread.csv.scraper.CsvScraper",
"signature": "CsvScraper(*, client: BaseCsvClient)",
"docstring": "Scraper for csv documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw csv bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n CSV content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.csv.scraper.CsvScraper.fetch",
"signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None) -> Content",
"docstring": "Fetch a csv document from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw csv bytes, source\n identifier, CSV content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client."
}
}
}
}
}
}

View File

@@ -8,29 +8,29 @@
"name": "HTMLScraper",
"kind": "class",
"path": "omniread.html.HTMLScraper",
"signature": "<bound method Alias.signature of Alias('HTMLScraper', 'omniread.html.scraper.HTMLScraper')>",
"signature": "HTMLScraper(*, client: httpx.Client | None = None, timeout: float = 15.0, headers: Mapping[str, str] | None = None, follow_redirects: bool = True)",
"docstring": "Base HTML scraper using `httpx`.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns\n them as raw content wrapped in a `Content` object.\n - Fetches raw bytes and metadata only.\n - The scraper uses `httpx.Client` for HTTP requests, enforces an\n HTML content type, and preserves HTTP response metadata.\n\n **Constraints:**\n\n - The scraper does not: Parse HTML, perform retries or backoff,\n handle non-HTML responses.",
"members": {
"content_type": {
"name": "content_type",
"kind": "attribute",
"path": "omniread.html.HTMLScraper.content_type",
"signature": "<bound method Alias.signature of Alias('content_type', 'omniread.html.scraper.HTMLScraper.content_type')>",
"signature": null,
"docstring": null
},
"validate_content_type": {
"name": "validate_content_type",
"kind": "function",
"path": "omniread.html.HTMLScraper.validate_content_type",
"signature": "<bound method Alias.signature of Alias('validate_content_type', 'omniread.html.scraper.HTMLScraper.validate_content_type')>",
"signature": "validate_content_type(response: httpx.Response)",
"docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content."
},
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.html.HTMLScraper.fetch",
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.html.scraper.HTMLScraper.fetch')>",
"docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML."
"signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None)",
"docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML."
}
}
},
@@ -38,49 +38,49 @@
"name": "HTMLParser",
"kind": "class",
"path": "omniread.html.HTMLParser",
"signature": "<bound method Alias.signature of Alias('HTMLParser', 'omniread.html.parser.HTMLParser')>",
"signature": "HTMLParser(content: Content, features: str = 'html.parser')",
"docstring": "Base HTML parser.\n\nNotes:\n **Responsibilities:**\n\n - This class extends the core `BaseParser` with HTML-specific behavior,\n including DOM parsing via BeautifulSoup and reusable extraction helpers.\n - Provides reusable helpers for HTML extraction. Concrete parsers must\n explicitly define the return type.\n\n **Guarantees:**\n\n - Accepts only HTML content.\n - Owns a parsed BeautifulSoup DOM tree.\n - Provides pure helper utilities for common HTML structures.\n\n **Constraints:**\n\n - Concrete subclasses must define the output type `T` and implement\n the `parse()` method.",
"members": {
"supported_types": {
"name": "supported_types",
"kind": "attribute",
"path": "omniread.html.HTMLParser.supported_types",
"signature": "<bound method Alias.signature of Alias('supported_types', 'omniread.html.parser.HTMLParser.supported_types')>",
"signature": null,
"docstring": "Set of content types supported by this parser (HTML only)."
},
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.html.HTMLParser.parse",
"signature": "<bound method Alias.signature of Alias('parse', 'omniread.html.parser.HTMLParser.parse')>",
"signature": "parse()",
"docstring": "Fully parse the HTML content into structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the HTML DOM and return a\n deterministic, structured output."
},
"parse_div": {
"name": "parse_div",
"kind": "function",
"path": "omniread.html.HTMLParser.parse_div",
"signature": "<bound method Alias.signature of Alias('parse_div', 'omniread.html.parser.HTMLParser.parse_div')>",
"signature": "parse_div(div: Tag, *, separator: str = ' ')",
"docstring": "Extract normalized text from a `<div>` element.\n\nArgs:\n div (Tag):\n BeautifulSoup tag representing a `<div>`.\n separator (str, optional):\n String used to separate text nodes.\n\nReturns:\n str:\n Flattened, whitespace-normalized text content."
},
"parse_link": {
"name": "parse_link",
"kind": "function",
"path": "omniread.html.HTMLParser.parse_link",
"signature": "<bound method Alias.signature of Alias('parse_link', 'omniread.html.parser.HTMLParser.parse_link')>",
"docstring": "Extract the hyperlink reference from an `<a>` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n Optional[str]:\n The value of the `href` attribute, or None if absent."
"signature": "parse_link(a: Tag)",
"docstring": "Extract the hyperlink reference from an `<a>` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n str | None:\n The value of the `href` attribute, or None if absent."
},
"parse_table": {
"name": "parse_table",
"kind": "function",
"path": "omniread.html.HTMLParser.parse_table",
"signature": "<bound method Alias.signature of Alias('parse_table', 'omniread.html.parser.HTMLParser.parse_table')>",
"signature": "parse_table(table: Tag)",
"docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table (Tag):\n BeautifulSoup tag representing a `<table>`.\n\nReturns:\n list[list[str]]:\n A list of rows, where each row is a list of cell text values."
},
"parse_meta": {
"name": "parse_meta",
"kind": "function",
"path": "omniread.html.HTMLParser.parse_meta",
"signature": "<bound method Alias.signature of Alias('parse_meta', 'omniread.html.parser.HTMLParser.parse_meta')>",
"signature": "parse_meta()",
"docstring": "Extract high-level metadata from the HTML document.\n\nReturns:\n dict[str, Any]:\n Dictionary containing extracted metadata.\n\nNotes:\n **Responsibilities:**\n\n - Extract high-level metadata from the HTML document.\n - This includes: Document title, `<meta>` tag name/property to\n content mappings."
}
}
@@ -92,162 +92,127 @@
"signature": null,
"docstring": "# Summary\n\nHTML parser base implementations for OmniRead.\n\nThis module provides reusable HTML parsing utilities built on top of\nthe abstract parser contracts defined in `omniread.core.parser`.\n\nIt supplies:\n\n- Content-type enforcement for HTML inputs\n- BeautifulSoup initialization and lifecycle management\n- Common helper methods for extracting structured data from HTML elements\n\nConcrete parsers must subclass `HTMLParser` and implement the `parse()` method\nto return a structured representation appropriate for their use case.",
"members": {
"Any": {
"name": "Any",
"kind": "alias",
"path": "omniread.html.parser.Any",
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
"docstring": null
},
"Generic": {
"name": "Generic",
"kind": "alias",
"path": "omniread.html.parser.Generic",
"signature": "<bound method Alias.signature of Alias('Generic', 'typing.Generic')>",
"docstring": null
},
"TypeVar": {
"name": "TypeVar",
"kind": "alias",
"path": "omniread.html.parser.TypeVar",
"signature": "<bound method Alias.signature of Alias('TypeVar', 'typing.TypeVar')>",
"docstring": null
},
"Optional": {
"name": "Optional",
"kind": "alias",
"path": "omniread.html.parser.Optional",
"signature": "<bound method Alias.signature of Alias('Optional', 'typing.Optional')>",
"docstring": null
},
"abstractmethod": {
"name": "abstractmethod",
"kind": "alias",
"path": "omniread.html.parser.abstractmethod",
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
"docstring": null
},
"BeautifulSoup": {
"name": "BeautifulSoup",
"kind": "alias",
"path": "omniread.html.parser.BeautifulSoup",
"signature": "<bound method Alias.signature of Alias('BeautifulSoup', 'bs4.BeautifulSoup')>",
"docstring": null
},
"Tag": {
"name": "Tag",
"kind": "alias",
"path": "omniread.html.parser.Tag",
"signature": "<bound method Alias.signature of Alias('Tag', 'bs4.Tag')>",
"docstring": null
},
"ContentType": {
"name": "ContentType",
"kind": "class",
"path": "omniread.html.parser.ContentType",
"signature": "<bound method Alias.signature of Alias('ContentType', 'omniread.core.content.ContentType')>",
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
"members": {
"HTML": {
"name": "HTML",
"kind": "attribute",
"path": "omniread.html.parser.ContentType.HTML",
"signature": "<bound method Alias.signature of Alias('HTML', 'omniread.core.content.ContentType.HTML')>",
"docstring": "HTML document content."
},
"PDF": {
"name": "PDF",
"kind": "attribute",
"path": "omniread.html.parser.ContentType.PDF",
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
"docstring": "PDF document content."
},
"JSON": {
"name": "JSON",
"kind": "attribute",
"path": "omniread.html.parser.ContentType.JSON",
"signature": "<bound method Alias.signature of Alias('JSON', 'omniread.core.content.ContentType.JSON')>",
"docstring": "JSON document content."
},
"XML": {
"name": "XML",
"kind": "attribute",
"path": "omniread.html.parser.ContentType.XML",
"signature": "<bound method Alias.signature of Alias('XML', 'omniread.core.content.ContentType.XML')>",
"docstring": "XML document content."
}
}
},
"Content": {
"name": "Content",
"kind": "class",
"path": "omniread.html.parser.Content",
"signature": "<bound method Alias.signature of Alias('Content', 'omniread.core.content.Content')>",
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
"members": {
"raw": {
"name": "raw",
"kind": "attribute",
"path": "omniread.html.parser.Content.raw",
"signature": "<bound method Alias.signature of Alias('raw', 'omniread.core.content.Content.raw')>",
"signature": null,
"docstring": "Raw content bytes as retrieved from the source."
},
"source": {
"name": "source",
"kind": "attribute",
"path": "omniread.html.parser.Content.source",
"signature": "<bound method Alias.signature of Alias('source', 'omniread.core.content.Content.source')>",
"signature": null,
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
},
"content_type": {
"name": "content_type",
"kind": "attribute",
"path": "omniread.html.parser.Content.content_type",
"signature": "<bound method Alias.signature of Alias('content_type', 'omniread.core.content.Content.content_type')>",
"signature": null,
"docstring": "Optional MIME type of the content, if known."
},
"metadata": {
"name": "metadata",
"kind": "attribute",
"path": "omniread.html.parser.Content.metadata",
"signature": "<bound method Alias.signature of Alias('metadata', 'omniread.core.content.Content.metadata')>",
"signature": null,
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
}
}
},
"ContentType": {
"name": "ContentType",
"kind": "class",
"path": "omniread.html.parser.ContentType",
"signature": null,
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
"members": {
"HTML": {
"name": "HTML",
"kind": "attribute",
"path": "omniread.html.parser.ContentType.HTML",
"signature": null,
"docstring": "HTML document content."
},
"PDF": {
"name": "PDF",
"kind": "attribute",
"path": "omniread.html.parser.ContentType.PDF",
"signature": null,
"docstring": "PDF document content."
},
"XLSX": {
"name": "XLSX",
"kind": "attribute",
"path": "omniread.html.parser.ContentType.XLSX",
"signature": null,
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
},
"CSV": {
"name": "CSV",
"kind": "attribute",
"path": "omniread.html.parser.ContentType.CSV",
"signature": null,
"docstring": "Comma-separated-value document content."
},
"JSON": {
"name": "JSON",
"kind": "attribute",
"path": "omniread.html.parser.ContentType.JSON",
"signature": null,
"docstring": "JSON document content."
},
"XML": {
"name": "XML",
"kind": "attribute",
"path": "omniread.html.parser.ContentType.XML",
"signature": null,
"docstring": "XML document content."
}
}
},
"BaseParser": {
"name": "BaseParser",
"kind": "class",
"path": "omniread.html.parser.BaseParser",
"signature": "<bound method Alias.signature of Alias('BaseParser', 'omniread.core.parser.BaseParser')>",
"signature": "BaseParser(content: Content)",
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
"members": {
"supported_types": {
"name": "supported_types",
"kind": "attribute",
"path": "omniread.html.parser.BaseParser.supported_types",
"signature": "<bound method Alias.signature of Alias('supported_types', 'omniread.core.parser.BaseParser.supported_types')>",
"signature": null,
"docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic."
},
"content": {
"name": "content",
"kind": "attribute",
"path": "omniread.html.parser.BaseParser.content",
"signature": "<bound method Alias.signature of Alias('content', 'omniread.core.parser.BaseParser.content')>",
"signature": null,
"docstring": null
},
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.html.parser.BaseParser.parse",
"signature": "<bound method Alias.signature of Alias('parse', 'omniread.core.parser.BaseParser.parse')>",
"signature": "parse()",
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
},
"supports": {
"name": "supports",
"kind": "function",
"path": "omniread.html.parser.BaseParser.supports",
"signature": "<bound method Alias.signature of Alias('supports', 'omniread.core.parser.BaseParser.supports')>",
"signature": "supports()",
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
}
}
@@ -263,7 +228,7 @@
"name": "HTMLParser",
"kind": "class",
"path": "omniread.html.parser.HTMLParser",
"signature": "<bound method Class.signature of Class('HTMLParser', 30, 205)>",
"signature": "HTMLParser(content: Content, features: str = 'html.parser')",
"docstring": "Base HTML parser.\n\nNotes:\n **Responsibilities:**\n\n - This class extends the core `BaseParser` with HTML-specific behavior,\n including DOM parsing via BeautifulSoup and reusable extraction helpers.\n - Provides reusable helpers for HTML extraction. Concrete parsers must\n explicitly define the return type.\n\n **Guarantees:**\n\n - Accepts only HTML content.\n - Owns a parsed BeautifulSoup DOM tree.\n - Provides pure helper utilities for common HTML structures.\n\n **Constraints:**\n\n - Concrete subclasses must define the output type `T` and implement\n the `parse()` method.",
"members": {
"supported_types": {
@@ -277,52 +242,38 @@
"name": "parse",
"kind": "function",
"path": "omniread.html.parser.HTMLParser.parse",
"signature": "<bound method Function.signature of Function('parse', 81, 96)>",
"signature": "parse() -> T",
"docstring": "Fully parse the HTML content into structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the HTML DOM and return a\n deterministic, structured output."
},
"parse_div": {
"name": "parse_div",
"kind": "function",
"path": "omniread.html.parser.HTMLParser.parse_div",
"signature": "<bound method Function.signature of Function('parse_div', 102, 117)>",
"signature": "parse_div(div: Tag, *, separator: str = ' ') -> str",
"docstring": "Extract normalized text from a `<div>` element.\n\nArgs:\n div (Tag):\n BeautifulSoup tag representing a `<div>`.\n separator (str, optional):\n String used to separate text nodes.\n\nReturns:\n str:\n Flattened, whitespace-normalized text content."
},
"parse_link": {
"name": "parse_link",
"kind": "function",
"path": "omniread.html.parser.HTMLParser.parse_link",
"signature": "<bound method Function.signature of Function('parse_link', 119, 132)>",
"docstring": "Extract the hyperlink reference from an `<a>` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n Optional[str]:\n The value of the `href` attribute, or None if absent."
"signature": "parse_link(a: Tag) -> str | None",
"docstring": "Extract the hyperlink reference from an `<a>` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n str | None:\n The value of the `href` attribute, or None if absent."
},
"parse_table": {
"name": "parse_table",
"kind": "function",
"path": "omniread.html.parser.HTMLParser.parse_table",
"signature": "<bound method Function.signature of Function('parse_table', 134, 155)>",
"signature": "parse_table(table: Tag) -> list[list[str]]",
"docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table (Tag):\n BeautifulSoup tag representing a `<table>`.\n\nReturns:\n list[list[str]]:\n A list of rows, where each row is a list of cell text values."
},
"parse_meta": {
"name": "parse_meta",
"kind": "function",
"path": "omniread.html.parser.HTMLParser.parse_meta",
"signature": "<bound method Function.signature of Function('parse_meta', 177, 205)>",
"signature": "parse_meta() -> dict[str, Any]",
"docstring": "Extract high-level metadata from the HTML document.\n\nReturns:\n dict[str, Any]:\n Dictionary containing extracted metadata.\n\nNotes:\n **Responsibilities:**\n\n - Extract high-level metadata from the HTML document.\n - This includes: Document title, `<meta>` tag name/property to\n content mappings."
}
}
},
"list": {
"name": "list",
"kind": "alias",
"path": "omniread.html.parser.list",
"signature": "<bound method Alias.signature of Alias('list', 'typing.list')>",
"docstring": null
},
"dict": {
"name": "dict",
"kind": "alias",
"path": "omniread.html.parser.dict",
"signature": "<bound method Alias.signature of Alias('dict', 'typing.dict')>",
"docstring": null
}
}
},
@@ -333,67 +284,39 @@
"signature": null,
"docstring": "# Summary\n\nHTML scraping implementation for OmniRead.\n\nThis module provides an HTTP-based scraper for retrieving HTML documents.\nIt implements the core `BaseScraper` contract using `httpx` as the transport\nlayer.\n\nThis scraper is responsible for:\n\n- Fetching raw HTML bytes over HTTP(S)\n- Validating response content type\n- Attaching HTTP metadata to the returned content\n\nThis scraper is not responsible for:\n\n- Parsing or interpreting HTML\n- Retrying failed requests\n- Managing crawl policies or rate limiting",
"members": {
"httpx": {
"name": "httpx",
"kind": "alias",
"path": "omniread.html.scraper.httpx",
"signature": "<bound method Alias.signature of Alias('httpx', 'httpx')>",
"docstring": null
},
"Any": {
"name": "Any",
"kind": "alias",
"path": "omniread.html.scraper.Any",
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
"docstring": null
},
"Mapping": {
"name": "Mapping",
"kind": "alias",
"path": "omniread.html.scraper.Mapping",
"signature": "<bound method Alias.signature of Alias('Mapping', 'typing.Mapping')>",
"docstring": null
},
"Optional": {
"name": "Optional",
"kind": "alias",
"path": "omniread.html.scraper.Optional",
"signature": "<bound method Alias.signature of Alias('Optional', 'typing.Optional')>",
"docstring": null
},
"Content": {
"name": "Content",
"kind": "class",
"path": "omniread.html.scraper.Content",
"signature": "<bound method Alias.signature of Alias('Content', 'omniread.core.content.Content')>",
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
"members": {
"raw": {
"name": "raw",
"kind": "attribute",
"path": "omniread.html.scraper.Content.raw",
"signature": "<bound method Alias.signature of Alias('raw', 'omniread.core.content.Content.raw')>",
"signature": null,
"docstring": "Raw content bytes as retrieved from the source."
},
"source": {
"name": "source",
"kind": "attribute",
"path": "omniread.html.scraper.Content.source",
"signature": "<bound method Alias.signature of Alias('source', 'omniread.core.content.Content.source')>",
"signature": null,
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
},
"content_type": {
"name": "content_type",
"kind": "attribute",
"path": "omniread.html.scraper.Content.content_type",
"signature": "<bound method Alias.signature of Alias('content_type', 'omniread.core.content.Content.content_type')>",
"signature": null,
"docstring": "Optional MIME type of the content, if known."
},
"metadata": {
"name": "metadata",
"kind": "attribute",
"path": "omniread.html.scraper.Content.metadata",
"signature": "<bound method Alias.signature of Alias('metadata', 'omniread.core.content.Content.metadata')>",
"signature": null,
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
}
}
@@ -402,35 +325,49 @@
"name": "ContentType",
"kind": "class",
"path": "omniread.html.scraper.ContentType",
"signature": "<bound method Alias.signature of Alias('ContentType', 'omniread.core.content.ContentType')>",
"signature": null,
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
"members": {
"HTML": {
"name": "HTML",
"kind": "attribute",
"path": "omniread.html.scraper.ContentType.HTML",
"signature": "<bound method Alias.signature of Alias('HTML', 'omniread.core.content.ContentType.HTML')>",
"signature": null,
"docstring": "HTML document content."
},
"PDF": {
"name": "PDF",
"kind": "attribute",
"path": "omniread.html.scraper.ContentType.PDF",
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
"signature": null,
"docstring": "PDF document content."
},
"XLSX": {
"name": "XLSX",
"kind": "attribute",
"path": "omniread.html.scraper.ContentType.XLSX",
"signature": null,
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
},
"CSV": {
"name": "CSV",
"kind": "attribute",
"path": "omniread.html.scraper.ContentType.CSV",
"signature": null,
"docstring": "Comma-separated-value document content."
},
"JSON": {
"name": "JSON",
"kind": "attribute",
"path": "omniread.html.scraper.ContentType.JSON",
"signature": "<bound method Alias.signature of Alias('JSON', 'omniread.core.content.ContentType.JSON')>",
"signature": null,
"docstring": "JSON document content."
},
"XML": {
"name": "XML",
"kind": "attribute",
"path": "omniread.html.scraper.ContentType.XML",
"signature": "<bound method Alias.signature of Alias('XML', 'omniread.core.content.ContentType.XML')>",
"signature": null,
"docstring": "XML document content."
}
}
@@ -439,15 +376,15 @@
"name": "BaseScraper",
"kind": "class",
"path": "omniread.html.scraper.BaseScraper",
"signature": "<bound method Alias.signature of Alias('BaseScraper', 'omniread.core.scraper.BaseScraper')>",
"signature": null,
"docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.html.scraper.BaseScraper.fetch",
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.core.scraper.BaseScraper.fetch')>",
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
"signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None)",
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
}
}
},
@@ -455,7 +392,7 @@
"name": "HTMLScraper",
"kind": "class",
"path": "omniread.html.scraper.HTMLScraper",
"signature": "<bound method Class.signature of Class('HTMLScraper', 30, 143)>",
"signature": "HTMLScraper(*, client: httpx.Client | None = None, timeout: float = 15.0, headers: Mapping[str, str] | None = None, follow_redirects: bool = True)",
"docstring": "Base HTML scraper using `httpx`.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns\n them as raw content wrapped in a `Content` object.\n - Fetches raw bytes and metadata only.\n - The scraper uses `httpx.Client` for HTTP requests, enforces an\n HTML content type, and preserves HTTP response metadata.\n\n **Constraints:**\n\n - The scraper does not: Parse HTML, perform retries or backoff,\n handle non-HTML responses.",
"members": {
"content_type": {
@@ -469,15 +406,15 @@
"name": "validate_content_type",
"kind": "function",
"path": "omniread.html.scraper.HTMLScraper.validate_content_type",
"signature": "<bound method Function.signature of Function('validate_content_type', 78, 102)>",
"signature": "validate_content_type(response: httpx.Response) -> None",
"docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content."
},
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.html.scraper.HTMLScraper.fetch",
"signature": "<bound method Function.signature of Function('fetch', 104, 143)>",
"docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML."
"signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None) -> Content",
"docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML."
}
}
}

View File

@@ -4,162 +4,127 @@
"path": "omniread.html.parser",
"docstring": "# Summary\n\nHTML parser base implementations for OmniRead.\n\nThis module provides reusable HTML parsing utilities built on top of\nthe abstract parser contracts defined in `omniread.core.parser`.\n\nIt supplies:\n\n- Content-type enforcement for HTML inputs\n- BeautifulSoup initialization and lifecycle management\n- Common helper methods for extracting structured data from HTML elements\n\nConcrete parsers must subclass `HTMLParser` and implement the `parse()` method\nto return a structured representation appropriate for their use case.",
"objects": {
"Any": {
"name": "Any",
"kind": "alias",
"path": "omniread.html.parser.Any",
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
"docstring": null
},
"Generic": {
"name": "Generic",
"kind": "alias",
"path": "omniread.html.parser.Generic",
"signature": "<bound method Alias.signature of Alias('Generic', 'typing.Generic')>",
"docstring": null
},
"TypeVar": {
"name": "TypeVar",
"kind": "alias",
"path": "omniread.html.parser.TypeVar",
"signature": "<bound method Alias.signature of Alias('TypeVar', 'typing.TypeVar')>",
"docstring": null
},
"Optional": {
"name": "Optional",
"kind": "alias",
"path": "omniread.html.parser.Optional",
"signature": "<bound method Alias.signature of Alias('Optional', 'typing.Optional')>",
"docstring": null
},
"abstractmethod": {
"name": "abstractmethod",
"kind": "alias",
"path": "omniread.html.parser.abstractmethod",
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
"docstring": null
},
"BeautifulSoup": {
"name": "BeautifulSoup",
"kind": "alias",
"path": "omniread.html.parser.BeautifulSoup",
"signature": "<bound method Alias.signature of Alias('BeautifulSoup', 'bs4.BeautifulSoup')>",
"docstring": null
},
"Tag": {
"name": "Tag",
"kind": "alias",
"path": "omniread.html.parser.Tag",
"signature": "<bound method Alias.signature of Alias('Tag', 'bs4.Tag')>",
"docstring": null
},
"ContentType": {
"name": "ContentType",
"kind": "class",
"path": "omniread.html.parser.ContentType",
"signature": "<bound method Alias.signature of Alias('ContentType', 'omniread.core.content.ContentType')>",
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
"members": {
"HTML": {
"name": "HTML",
"kind": "attribute",
"path": "omniread.html.parser.ContentType.HTML",
"signature": "<bound method Alias.signature of Alias('HTML', 'omniread.core.content.ContentType.HTML')>",
"docstring": "HTML document content."
},
"PDF": {
"name": "PDF",
"kind": "attribute",
"path": "omniread.html.parser.ContentType.PDF",
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
"docstring": "PDF document content."
},
"JSON": {
"name": "JSON",
"kind": "attribute",
"path": "omniread.html.parser.ContentType.JSON",
"signature": "<bound method Alias.signature of Alias('JSON', 'omniread.core.content.ContentType.JSON')>",
"docstring": "JSON document content."
},
"XML": {
"name": "XML",
"kind": "attribute",
"path": "omniread.html.parser.ContentType.XML",
"signature": "<bound method Alias.signature of Alias('XML', 'omniread.core.content.ContentType.XML')>",
"docstring": "XML document content."
}
}
},
"Content": {
"name": "Content",
"kind": "class",
"path": "omniread.html.parser.Content",
"signature": "<bound method Alias.signature of Alias('Content', 'omniread.core.content.Content')>",
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
"members": {
"raw": {
"name": "raw",
"kind": "attribute",
"path": "omniread.html.parser.Content.raw",
"signature": "<bound method Alias.signature of Alias('raw', 'omniread.core.content.Content.raw')>",
"signature": null,
"docstring": "Raw content bytes as retrieved from the source."
},
"source": {
"name": "source",
"kind": "attribute",
"path": "omniread.html.parser.Content.source",
"signature": "<bound method Alias.signature of Alias('source', 'omniread.core.content.Content.source')>",
"signature": null,
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
},
"content_type": {
"name": "content_type",
"kind": "attribute",
"path": "omniread.html.parser.Content.content_type",
"signature": "<bound method Alias.signature of Alias('content_type', 'omniread.core.content.Content.content_type')>",
"signature": null,
"docstring": "Optional MIME type of the content, if known."
},
"metadata": {
"name": "metadata",
"kind": "attribute",
"path": "omniread.html.parser.Content.metadata",
"signature": "<bound method Alias.signature of Alias('metadata', 'omniread.core.content.Content.metadata')>",
"signature": null,
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
}
}
},
"ContentType": {
"name": "ContentType",
"kind": "class",
"path": "omniread.html.parser.ContentType",
"signature": null,
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
"members": {
"HTML": {
"name": "HTML",
"kind": "attribute",
"path": "omniread.html.parser.ContentType.HTML",
"signature": null,
"docstring": "HTML document content."
},
"PDF": {
"name": "PDF",
"kind": "attribute",
"path": "omniread.html.parser.ContentType.PDF",
"signature": null,
"docstring": "PDF document content."
},
"XLSX": {
"name": "XLSX",
"kind": "attribute",
"path": "omniread.html.parser.ContentType.XLSX",
"signature": null,
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
},
"CSV": {
"name": "CSV",
"kind": "attribute",
"path": "omniread.html.parser.ContentType.CSV",
"signature": null,
"docstring": "Comma-separated-value document content."
},
"JSON": {
"name": "JSON",
"kind": "attribute",
"path": "omniread.html.parser.ContentType.JSON",
"signature": null,
"docstring": "JSON document content."
},
"XML": {
"name": "XML",
"kind": "attribute",
"path": "omniread.html.parser.ContentType.XML",
"signature": null,
"docstring": "XML document content."
}
}
},
"BaseParser": {
"name": "BaseParser",
"kind": "class",
"path": "omniread.html.parser.BaseParser",
"signature": "<bound method Alias.signature of Alias('BaseParser', 'omniread.core.parser.BaseParser')>",
"signature": "BaseParser(content: Content)",
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
"members": {
"supported_types": {
"name": "supported_types",
"kind": "attribute",
"path": "omniread.html.parser.BaseParser.supported_types",
"signature": "<bound method Alias.signature of Alias('supported_types', 'omniread.core.parser.BaseParser.supported_types')>",
"signature": null,
"docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic."
},
"content": {
"name": "content",
"kind": "attribute",
"path": "omniread.html.parser.BaseParser.content",
"signature": "<bound method Alias.signature of Alias('content', 'omniread.core.parser.BaseParser.content')>",
"signature": null,
"docstring": null
},
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.html.parser.BaseParser.parse",
"signature": "<bound method Alias.signature of Alias('parse', 'omniread.core.parser.BaseParser.parse')>",
"signature": "parse()",
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
},
"supports": {
"name": "supports",
"kind": "function",
"path": "omniread.html.parser.BaseParser.supports",
"signature": "<bound method Alias.signature of Alias('supports', 'omniread.core.parser.BaseParser.supports')>",
"signature": "supports()",
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
}
}
@@ -175,7 +140,7 @@
"name": "HTMLParser",
"kind": "class",
"path": "omniread.html.parser.HTMLParser",
"signature": "<bound method Class.signature of Class('HTMLParser', 30, 205)>",
"signature": "HTMLParser(content: Content, features: str = 'html.parser')",
"docstring": "Base HTML parser.\n\nNotes:\n **Responsibilities:**\n\n - This class extends the core `BaseParser` with HTML-specific behavior,\n including DOM parsing via BeautifulSoup and reusable extraction helpers.\n - Provides reusable helpers for HTML extraction. Concrete parsers must\n explicitly define the return type.\n\n **Guarantees:**\n\n - Accepts only HTML content.\n - Owns a parsed BeautifulSoup DOM tree.\n - Provides pure helper utilities for common HTML structures.\n\n **Constraints:**\n\n - Concrete subclasses must define the output type `T` and implement\n the `parse()` method.",
"members": {
"supported_types": {
@@ -189,52 +154,38 @@
"name": "parse",
"kind": "function",
"path": "omniread.html.parser.HTMLParser.parse",
"signature": "<bound method Function.signature of Function('parse', 81, 96)>",
"signature": "parse() -> T",
"docstring": "Fully parse the HTML content into structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the HTML DOM and return a\n deterministic, structured output."
},
"parse_div": {
"name": "parse_div",
"kind": "function",
"path": "omniread.html.parser.HTMLParser.parse_div",
"signature": "<bound method Function.signature of Function('parse_div', 102, 117)>",
"signature": "parse_div(div: Tag, *, separator: str = ' ') -> str",
"docstring": "Extract normalized text from a `<div>` element.\n\nArgs:\n div (Tag):\n BeautifulSoup tag representing a `<div>`.\n separator (str, optional):\n String used to separate text nodes.\n\nReturns:\n str:\n Flattened, whitespace-normalized text content."
},
"parse_link": {
"name": "parse_link",
"kind": "function",
"path": "omniread.html.parser.HTMLParser.parse_link",
"signature": "<bound method Function.signature of Function('parse_link', 119, 132)>",
"docstring": "Extract the hyperlink reference from an `<a>` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n Optional[str]:\n The value of the `href` attribute, or None if absent."
"signature": "parse_link(a: Tag) -> str | None",
"docstring": "Extract the hyperlink reference from an `<a>` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n str | None:\n The value of the `href` attribute, or None if absent."
},
"parse_table": {
"name": "parse_table",
"kind": "function",
"path": "omniread.html.parser.HTMLParser.parse_table",
"signature": "<bound method Function.signature of Function('parse_table', 134, 155)>",
"signature": "parse_table(table: Tag) -> list[list[str]]",
"docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table (Tag):\n BeautifulSoup tag representing a `<table>`.\n\nReturns:\n list[list[str]]:\n A list of rows, where each row is a list of cell text values."
},
"parse_meta": {
"name": "parse_meta",
"kind": "function",
"path": "omniread.html.parser.HTMLParser.parse_meta",
"signature": "<bound method Function.signature of Function('parse_meta', 177, 205)>",
"signature": "parse_meta() -> dict[str, Any]",
"docstring": "Extract high-level metadata from the HTML document.\n\nReturns:\n dict[str, Any]:\n Dictionary containing extracted metadata.\n\nNotes:\n **Responsibilities:**\n\n - Extract high-level metadata from the HTML document.\n - This includes: Document title, `<meta>` tag name/property to\n content mappings."
}
}
},
"list": {
"name": "list",
"kind": "alias",
"path": "omniread.html.parser.list",
"signature": "<bound method Alias.signature of Alias('list', 'typing.list')>",
"docstring": null
},
"dict": {
"name": "dict",
"kind": "alias",
"path": "omniread.html.parser.dict",
"signature": "<bound method Alias.signature of Alias('dict', 'typing.dict')>",
"docstring": null
}
}
}

View File

@@ -4,67 +4,39 @@
"path": "omniread.html.scraper",
"docstring": "# Summary\n\nHTML scraping implementation for OmniRead.\n\nThis module provides an HTTP-based scraper for retrieving HTML documents.\nIt implements the core `BaseScraper` contract using `httpx` as the transport\nlayer.\n\nThis scraper is responsible for:\n\n- Fetching raw HTML bytes over HTTP(S)\n- Validating response content type\n- Attaching HTTP metadata to the returned content\n\nThis scraper is not responsible for:\n\n- Parsing or interpreting HTML\n- Retrying failed requests\n- Managing crawl policies or rate limiting",
"objects": {
"httpx": {
"name": "httpx",
"kind": "alias",
"path": "omniread.html.scraper.httpx",
"signature": "<bound method Alias.signature of Alias('httpx', 'httpx')>",
"docstring": null
},
"Any": {
"name": "Any",
"kind": "alias",
"path": "omniread.html.scraper.Any",
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
"docstring": null
},
"Mapping": {
"name": "Mapping",
"kind": "alias",
"path": "omniread.html.scraper.Mapping",
"signature": "<bound method Alias.signature of Alias('Mapping', 'typing.Mapping')>",
"docstring": null
},
"Optional": {
"name": "Optional",
"kind": "alias",
"path": "omniread.html.scraper.Optional",
"signature": "<bound method Alias.signature of Alias('Optional', 'typing.Optional')>",
"docstring": null
},
"Content": {
"name": "Content",
"kind": "class",
"path": "omniread.html.scraper.Content",
"signature": "<bound method Alias.signature of Alias('Content', 'omniread.core.content.Content')>",
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
"members": {
"raw": {
"name": "raw",
"kind": "attribute",
"path": "omniread.html.scraper.Content.raw",
"signature": "<bound method Alias.signature of Alias('raw', 'omniread.core.content.Content.raw')>",
"signature": null,
"docstring": "Raw content bytes as retrieved from the source."
},
"source": {
"name": "source",
"kind": "attribute",
"path": "omniread.html.scraper.Content.source",
"signature": "<bound method Alias.signature of Alias('source', 'omniread.core.content.Content.source')>",
"signature": null,
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
},
"content_type": {
"name": "content_type",
"kind": "attribute",
"path": "omniread.html.scraper.Content.content_type",
"signature": "<bound method Alias.signature of Alias('content_type', 'omniread.core.content.Content.content_type')>",
"signature": null,
"docstring": "Optional MIME type of the content, if known."
},
"metadata": {
"name": "metadata",
"kind": "attribute",
"path": "omniread.html.scraper.Content.metadata",
"signature": "<bound method Alias.signature of Alias('metadata', 'omniread.core.content.Content.metadata')>",
"signature": null,
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
}
}
@@ -73,35 +45,49 @@
"name": "ContentType",
"kind": "class",
"path": "omniread.html.scraper.ContentType",
"signature": "<bound method Alias.signature of Alias('ContentType', 'omniread.core.content.ContentType')>",
"signature": null,
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
"members": {
"HTML": {
"name": "HTML",
"kind": "attribute",
"path": "omniread.html.scraper.ContentType.HTML",
"signature": "<bound method Alias.signature of Alias('HTML', 'omniread.core.content.ContentType.HTML')>",
"signature": null,
"docstring": "HTML document content."
},
"PDF": {
"name": "PDF",
"kind": "attribute",
"path": "omniread.html.scraper.ContentType.PDF",
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
"signature": null,
"docstring": "PDF document content."
},
"XLSX": {
"name": "XLSX",
"kind": "attribute",
"path": "omniread.html.scraper.ContentType.XLSX",
"signature": null,
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
},
"CSV": {
"name": "CSV",
"kind": "attribute",
"path": "omniread.html.scraper.ContentType.CSV",
"signature": null,
"docstring": "Comma-separated-value document content."
},
"JSON": {
"name": "JSON",
"kind": "attribute",
"path": "omniread.html.scraper.ContentType.JSON",
"signature": "<bound method Alias.signature of Alias('JSON', 'omniread.core.content.ContentType.JSON')>",
"signature": null,
"docstring": "JSON document content."
},
"XML": {
"name": "XML",
"kind": "attribute",
"path": "omniread.html.scraper.ContentType.XML",
"signature": "<bound method Alias.signature of Alias('XML', 'omniread.core.content.ContentType.XML')>",
"signature": null,
"docstring": "XML document content."
}
}
@@ -110,15 +96,15 @@
"name": "BaseScraper",
"kind": "class",
"path": "omniread.html.scraper.BaseScraper",
"signature": "<bound method Alias.signature of Alias('BaseScraper', 'omniread.core.scraper.BaseScraper')>",
"signature": null,
"docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.html.scraper.BaseScraper.fetch",
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.core.scraper.BaseScraper.fetch')>",
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
"signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None)",
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
}
}
},
@@ -126,7 +112,7 @@
"name": "HTMLScraper",
"kind": "class",
"path": "omniread.html.scraper.HTMLScraper",
"signature": "<bound method Class.signature of Class('HTMLScraper', 30, 143)>",
"signature": "HTMLScraper(*, client: httpx.Client | None = None, timeout: float = 15.0, headers: Mapping[str, str] | None = None, follow_redirects: bool = True)",
"docstring": "Base HTML scraper using `httpx`.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns\n them as raw content wrapped in a `Content` object.\n - Fetches raw bytes and metadata only.\n - The scraper uses `httpx.Client` for HTTP requests, enforces an\n HTML content type, and preserves HTTP response metadata.\n\n **Constraints:**\n\n - The scraper does not: Parse HTML, perform retries or backoff,\n handle non-HTML responses.",
"members": {
"content_type": {
@@ -140,15 +126,15 @@
"name": "validate_content_type",
"kind": "function",
"path": "omniread.html.scraper.HTMLScraper.validate_content_type",
"signature": "<bound method Function.signature of Function('validate_content_type', 78, 102)>",
"signature": "validate_content_type(response: httpx.Response) -> None",
"docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content."
},
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.html.scraper.HTMLScraper.fetch",
"signature": "<bound method Function.signature of Function('fetch', 104, 143)>",
"docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML."
"signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None) -> Content",
"docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML."
}
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -4,46 +4,18 @@
"path": "omniread.pdf.client",
"docstring": "# Summary\n\nPDF client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw PDF\nbytes from a concrete backing store.\n\nClients provide low-level access to PDF binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems",
"objects": {
"Any": {
"name": "Any",
"kind": "alias",
"path": "omniread.pdf.client.Any",
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
"docstring": null
},
"ABC": {
"name": "ABC",
"kind": "alias",
"path": "omniread.pdf.client.ABC",
"signature": "<bound method Alias.signature of Alias('ABC', 'abc.ABC')>",
"docstring": null
},
"abstractmethod": {
"name": "abstractmethod",
"kind": "alias",
"path": "omniread.pdf.client.abstractmethod",
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
"docstring": null
},
"Path": {
"name": "Path",
"kind": "alias",
"path": "omniread.pdf.client.Path",
"signature": "<bound method Alias.signature of Alias('Path', 'pathlib.Path')>",
"docstring": null
},
"BasePDFClient": {
"name": "BasePDFClient",
"kind": "class",
"path": "omniread.pdf.client.BasePDFClient",
"signature": "<bound method Class.signature of Class('BasePDFClient', 25, 57)>",
"signature": null,
"docstring": "Abstract client responsible for retrieving PDF bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full PDF binary payload.\n - Raise retrieval-specific errors on failure.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.pdf.client.BasePDFClient.fetch",
"signature": "<bound method Function.signature of Function('fetch', 40, 57)>",
"signature": "fetch(source: Any) -> bytes",
"docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF location, such as a file path, object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
}
}
@@ -52,14 +24,14 @@
"name": "FileSystemPDFClient",
"kind": "class",
"path": "omniread.pdf.client.FileSystemPDFClient",
"signature": "<bound method Class.signature of Class('FileSystemPDFClient', 60, 96)>",
"signature": null,
"docstring": "PDF client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads PDF files directly from the disk and returns\n their raw binary contents.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.pdf.client.FileSystemPDFClient.fetch",
"signature": "<bound method Function.signature of Function('fetch', 71, 96)>",
"signature": "fetch(path: Path) -> bytes",
"docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the PDF file.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file."
}
}

View File

@@ -8,14 +8,14 @@
"name": "FileSystemPDFClient",
"kind": "class",
"path": "omniread.pdf.FileSystemPDFClient",
"signature": "<bound method Alias.signature of Alias('FileSystemPDFClient', 'omniread.pdf.client.FileSystemPDFClient')>",
"signature": null,
"docstring": "PDF client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads PDF files directly from the disk and returns\n their raw binary contents.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.pdf.FileSystemPDFClient.fetch",
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.pdf.client.FileSystemPDFClient.fetch')>",
"signature": "fetch(path: Path)",
"docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the PDF file.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file."
}
}
@@ -24,15 +24,15 @@
"name": "PDFScraper",
"kind": "class",
"path": "omniread.pdf.PDFScraper",
"signature": "<bound method Alias.signature of Alias('PDFScraper', 'omniread.pdf.scraper.PDFScraper')>",
"signature": "PDFScraper(*, client: BasePDFClient)",
"docstring": "Scraper for PDF sources.\n\nNotes:\n **Responsibilities:**\n\n - Delegates byte retrieval to a PDF client and normalizes output\n into `Content`.\n - Preserves caller-provided metadata.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.pdf.PDFScraper.fetch",
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.pdf.scraper.PDFScraper.fetch')>",
"docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client."
"signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None)",
"docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client."
}
}
},
@@ -40,21 +40,21 @@
"name": "PDFParser",
"kind": "class",
"path": "omniread.pdf.PDFParser",
"signature": "<bound method Alias.signature of Alias('PDFParser', 'omniread.pdf.parser.PDFParser')>",
"signature": null,
"docstring": "Base PDF parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces PDF content-type compatibility and provides\n the extension point for implementing concrete PDF parsing strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
"members": {
"supported_types": {
"name": "supported_types",
"kind": "attribute",
"path": "omniread.pdf.PDFParser.supported_types",
"signature": "<bound method Alias.signature of Alias('supported_types', 'omniread.pdf.parser.PDFParser.supported_types')>",
"signature": null,
"docstring": "Set of content types supported by this parser (PDF only)."
},
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.pdf.PDFParser.parse",
"signature": "<bound method Alias.signature of Alias('parse', 'omniread.pdf.parser.PDFParser.parse')>",
"signature": "parse()",
"docstring": "Parse PDF content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the PDF binary payload and\n return a deterministic, structured output."
}
}
@@ -66,46 +66,18 @@
"signature": null,
"docstring": "# Summary\n\nPDF client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw PDF\nbytes from a concrete backing store.\n\nClients provide low-level access to PDF binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems",
"members": {
"Any": {
"name": "Any",
"kind": "alias",
"path": "omniread.pdf.client.Any",
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
"docstring": null
},
"ABC": {
"name": "ABC",
"kind": "alias",
"path": "omniread.pdf.client.ABC",
"signature": "<bound method Alias.signature of Alias('ABC', 'abc.ABC')>",
"docstring": null
},
"abstractmethod": {
"name": "abstractmethod",
"kind": "alias",
"path": "omniread.pdf.client.abstractmethod",
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
"docstring": null
},
"Path": {
"name": "Path",
"kind": "alias",
"path": "omniread.pdf.client.Path",
"signature": "<bound method Alias.signature of Alias('Path', 'pathlib.Path')>",
"docstring": null
},
"BasePDFClient": {
"name": "BasePDFClient",
"kind": "class",
"path": "omniread.pdf.client.BasePDFClient",
"signature": "<bound method Class.signature of Class('BasePDFClient', 25, 57)>",
"signature": null,
"docstring": "Abstract client responsible for retrieving PDF bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full PDF binary payload.\n - Raise retrieval-specific errors on failure.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.pdf.client.BasePDFClient.fetch",
"signature": "<bound method Function.signature of Function('fetch', 40, 57)>",
"signature": "fetch(source: Any) -> bytes",
"docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF location, such as a file path, object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
}
}
@@ -114,14 +86,14 @@
"name": "FileSystemPDFClient",
"kind": "class",
"path": "omniread.pdf.client.FileSystemPDFClient",
"signature": "<bound method Class.signature of Class('FileSystemPDFClient', 60, 96)>",
"signature": null,
"docstring": "PDF client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads PDF files directly from the disk and returns\n their raw binary contents.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.pdf.client.FileSystemPDFClient.fetch",
"signature": "<bound method Function.signature of Function('fetch', 71, 96)>",
"signature": "fetch(path: Path) -> bytes",
"docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the PDF file.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file."
}
}
@@ -135,60 +107,53 @@
"signature": null,
"docstring": "# Summary\n\nPDF parser base implementations for OmniRead.\n\nThis module defines the **PDF-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for PDF content.\n\nPDF parsers are responsible for interpreting binary PDF data and producing\nstructured representations suitable for downstream consumption.",
"members": {
"Generic": {
"name": "Generic",
"kind": "alias",
"path": "omniread.pdf.parser.Generic",
"signature": "<bound method Alias.signature of Alias('Generic', 'typing.Generic')>",
"docstring": null
},
"TypeVar": {
"name": "TypeVar",
"kind": "alias",
"path": "omniread.pdf.parser.TypeVar",
"signature": "<bound method Alias.signature of Alias('TypeVar', 'typing.TypeVar')>",
"docstring": null
},
"abstractmethod": {
"name": "abstractmethod",
"kind": "alias",
"path": "omniread.pdf.parser.abstractmethod",
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
"docstring": null
},
"ContentType": {
"name": "ContentType",
"kind": "class",
"path": "omniread.pdf.parser.ContentType",
"signature": "<bound method Alias.signature of Alias('ContentType', 'omniread.core.content.ContentType')>",
"signature": null,
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
"members": {
"HTML": {
"name": "HTML",
"kind": "attribute",
"path": "omniread.pdf.parser.ContentType.HTML",
"signature": "<bound method Alias.signature of Alias('HTML', 'omniread.core.content.ContentType.HTML')>",
"signature": null,
"docstring": "HTML document content."
},
"PDF": {
"name": "PDF",
"kind": "attribute",
"path": "omniread.pdf.parser.ContentType.PDF",
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
"signature": null,
"docstring": "PDF document content."
},
"XLSX": {
"name": "XLSX",
"kind": "attribute",
"path": "omniread.pdf.parser.ContentType.XLSX",
"signature": null,
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
},
"CSV": {
"name": "CSV",
"kind": "attribute",
"path": "omniread.pdf.parser.ContentType.CSV",
"signature": null,
"docstring": "Comma-separated-value document content."
},
"JSON": {
"name": "JSON",
"kind": "attribute",
"path": "omniread.pdf.parser.ContentType.JSON",
"signature": "<bound method Alias.signature of Alias('JSON', 'omniread.core.content.ContentType.JSON')>",
"signature": null,
"docstring": "JSON document content."
},
"XML": {
"name": "XML",
"kind": "attribute",
"path": "omniread.pdf.parser.ContentType.XML",
"signature": "<bound method Alias.signature of Alias('XML', 'omniread.core.content.ContentType.XML')>",
"signature": null,
"docstring": "XML document content."
}
}
@@ -197,35 +162,35 @@
"name": "BaseParser",
"kind": "class",
"path": "omniread.pdf.parser.BaseParser",
"signature": "<bound method Alias.signature of Alias('BaseParser', 'omniread.core.parser.BaseParser')>",
"signature": "BaseParser(content: Content)",
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
"members": {
"supported_types": {
"name": "supported_types",
"kind": "attribute",
"path": "omniread.pdf.parser.BaseParser.supported_types",
"signature": "<bound method Alias.signature of Alias('supported_types', 'omniread.core.parser.BaseParser.supported_types')>",
"signature": null,
"docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic."
},
"content": {
"name": "content",
"kind": "attribute",
"path": "omniread.pdf.parser.BaseParser.content",
"signature": "<bound method Alias.signature of Alias('content', 'omniread.core.parser.BaseParser.content')>",
"signature": null,
"docstring": null
},
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.pdf.parser.BaseParser.parse",
"signature": "<bound method Alias.signature of Alias('parse', 'omniread.core.parser.BaseParser.parse')>",
"signature": "parse()",
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
},
"supports": {
"name": "supports",
"kind": "function",
"path": "omniread.pdf.parser.BaseParser.supports",
"signature": "<bound method Alias.signature of Alias('supports', 'omniread.core.parser.BaseParser.supports')>",
"signature": "supports()",
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
}
}
@@ -241,7 +206,7 @@
"name": "PDFParser",
"kind": "class",
"path": "omniread.pdf.parser.PDFParser",
"signature": "<bound method Class.signature of Class('PDFParser', 22, 62)>",
"signature": null,
"docstring": "Base PDF parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces PDF content-type compatibility and provides\n the extension point for implementing concrete PDF parsing strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
"members": {
"supported_types": {
@@ -255,7 +220,7 @@
"name": "parse",
"kind": "function",
"path": "omniread.pdf.parser.PDFParser.parse",
"signature": "<bound method Function.signature of Function('parse', 43, 62)>",
"signature": "parse() -> T",
"docstring": "Parse PDF content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the PDF binary payload and\n return a deterministic, structured output."
}
}
@@ -269,60 +234,39 @@
"signature": null,
"docstring": "# Summary\n\nPDF scraping implementation for OmniRead.\n\nThis module provides a PDF-specific scraper that coordinates PDF byte\nretrieval via a client and normalizes the result into a `Content` object.\n\nThe scraper implements the core `BaseScraper` contract while delegating\nall storage and access concerns to a `BasePDFClient` implementation.",
"members": {
"Any": {
"name": "Any",
"kind": "alias",
"path": "omniread.pdf.scraper.Any",
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
"docstring": null
},
"Mapping": {
"name": "Mapping",
"kind": "alias",
"path": "omniread.pdf.scraper.Mapping",
"signature": "<bound method Alias.signature of Alias('Mapping', 'typing.Mapping')>",
"docstring": null
},
"Optional": {
"name": "Optional",
"kind": "alias",
"path": "omniread.pdf.scraper.Optional",
"signature": "<bound method Alias.signature of Alias('Optional', 'typing.Optional')>",
"docstring": null
},
"Content": {
"name": "Content",
"kind": "class",
"path": "omniread.pdf.scraper.Content",
"signature": "<bound method Alias.signature of Alias('Content', 'omniread.core.content.Content')>",
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
"members": {
"raw": {
"name": "raw",
"kind": "attribute",
"path": "omniread.pdf.scraper.Content.raw",
"signature": "<bound method Alias.signature of Alias('raw', 'omniread.core.content.Content.raw')>",
"signature": null,
"docstring": "Raw content bytes as retrieved from the source."
},
"source": {
"name": "source",
"kind": "attribute",
"path": "omniread.pdf.scraper.Content.source",
"signature": "<bound method Alias.signature of Alias('source', 'omniread.core.content.Content.source')>",
"signature": null,
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
},
"content_type": {
"name": "content_type",
"kind": "attribute",
"path": "omniread.pdf.scraper.Content.content_type",
"signature": "<bound method Alias.signature of Alias('content_type', 'omniread.core.content.Content.content_type')>",
"signature": null,
"docstring": "Optional MIME type of the content, if known."
},
"metadata": {
"name": "metadata",
"kind": "attribute",
"path": "omniread.pdf.scraper.Content.metadata",
"signature": "<bound method Alias.signature of Alias('metadata', 'omniread.core.content.Content.metadata')>",
"signature": null,
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
}
}
@@ -331,35 +275,49 @@
"name": "ContentType",
"kind": "class",
"path": "omniread.pdf.scraper.ContentType",
"signature": "<bound method Alias.signature of Alias('ContentType', 'omniread.core.content.ContentType')>",
"signature": null,
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
"members": {
"HTML": {
"name": "HTML",
"kind": "attribute",
"path": "omniread.pdf.scraper.ContentType.HTML",
"signature": "<bound method Alias.signature of Alias('HTML', 'omniread.core.content.ContentType.HTML')>",
"signature": null,
"docstring": "HTML document content."
},
"PDF": {
"name": "PDF",
"kind": "attribute",
"path": "omniread.pdf.scraper.ContentType.PDF",
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
"signature": null,
"docstring": "PDF document content."
},
"XLSX": {
"name": "XLSX",
"kind": "attribute",
"path": "omniread.pdf.scraper.ContentType.XLSX",
"signature": null,
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
},
"CSV": {
"name": "CSV",
"kind": "attribute",
"path": "omniread.pdf.scraper.ContentType.CSV",
"signature": null,
"docstring": "Comma-separated-value document content."
},
"JSON": {
"name": "JSON",
"kind": "attribute",
"path": "omniread.pdf.scraper.ContentType.JSON",
"signature": "<bound method Alias.signature of Alias('JSON', 'omniread.core.content.ContentType.JSON')>",
"signature": null,
"docstring": "JSON document content."
},
"XML": {
"name": "XML",
"kind": "attribute",
"path": "omniread.pdf.scraper.ContentType.XML",
"signature": "<bound method Alias.signature of Alias('XML', 'omniread.core.content.ContentType.XML')>",
"signature": null,
"docstring": "XML document content."
}
}
@@ -368,15 +326,15 @@
"name": "BaseScraper",
"kind": "class",
"path": "omniread.pdf.scraper.BaseScraper",
"signature": "<bound method Alias.signature of Alias('BaseScraper', 'omniread.core.scraper.BaseScraper')>",
"signature": null,
"docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.pdf.scraper.BaseScraper.fetch",
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.core.scraper.BaseScraper.fetch')>",
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
"signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None)",
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
}
}
},
@@ -384,14 +342,14 @@
"name": "BasePDFClient",
"kind": "class",
"path": "omniread.pdf.scraper.BasePDFClient",
"signature": "<bound method Alias.signature of Alias('BasePDFClient', 'omniread.pdf.client.BasePDFClient')>",
"signature": null,
"docstring": "Abstract client responsible for retrieving PDF bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full PDF binary payload.\n - Raise retrieval-specific errors on failure.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.pdf.scraper.BasePDFClient.fetch",
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.pdf.client.BasePDFClient.fetch')>",
"signature": "fetch(source: Any)",
"docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF location, such as a file path, object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
}
}
@@ -400,15 +358,15 @@
"name": "PDFScraper",
"kind": "class",
"path": "omniread.pdf.scraper.PDFScraper",
"signature": "<bound method Class.signature of Class('PDFScraper', 20, 77)>",
"signature": "PDFScraper(*, client: BasePDFClient)",
"docstring": "Scraper for PDF sources.\n\nNotes:\n **Responsibilities:**\n\n - Delegates byte retrieval to a PDF client and normalizes output\n into `Content`.\n - Preserves caller-provided metadata.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.pdf.scraper.PDFScraper.fetch",
"signature": "<bound method Function.signature of Function('fetch', 47, 77)>",
"docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client."
"signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None) -> Content",
"docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client."
}
}
}

View File

@@ -4,60 +4,53 @@
"path": "omniread.pdf.parser",
"docstring": "# Summary\n\nPDF parser base implementations for OmniRead.\n\nThis module defines the **PDF-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for PDF content.\n\nPDF parsers are responsible for interpreting binary PDF data and producing\nstructured representations suitable for downstream consumption.",
"objects": {
"Generic": {
"name": "Generic",
"kind": "alias",
"path": "omniread.pdf.parser.Generic",
"signature": "<bound method Alias.signature of Alias('Generic', 'typing.Generic')>",
"docstring": null
},
"TypeVar": {
"name": "TypeVar",
"kind": "alias",
"path": "omniread.pdf.parser.TypeVar",
"signature": "<bound method Alias.signature of Alias('TypeVar', 'typing.TypeVar')>",
"docstring": null
},
"abstractmethod": {
"name": "abstractmethod",
"kind": "alias",
"path": "omniread.pdf.parser.abstractmethod",
"signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
"docstring": null
},
"ContentType": {
"name": "ContentType",
"kind": "class",
"path": "omniread.pdf.parser.ContentType",
"signature": "<bound method Alias.signature of Alias('ContentType', 'omniread.core.content.ContentType')>",
"signature": null,
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
"members": {
"HTML": {
"name": "HTML",
"kind": "attribute",
"path": "omniread.pdf.parser.ContentType.HTML",
"signature": "<bound method Alias.signature of Alias('HTML', 'omniread.core.content.ContentType.HTML')>",
"signature": null,
"docstring": "HTML document content."
},
"PDF": {
"name": "PDF",
"kind": "attribute",
"path": "omniread.pdf.parser.ContentType.PDF",
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
"signature": null,
"docstring": "PDF document content."
},
"XLSX": {
"name": "XLSX",
"kind": "attribute",
"path": "omniread.pdf.parser.ContentType.XLSX",
"signature": null,
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
},
"CSV": {
"name": "CSV",
"kind": "attribute",
"path": "omniread.pdf.parser.ContentType.CSV",
"signature": null,
"docstring": "Comma-separated-value document content."
},
"JSON": {
"name": "JSON",
"kind": "attribute",
"path": "omniread.pdf.parser.ContentType.JSON",
"signature": "<bound method Alias.signature of Alias('JSON', 'omniread.core.content.ContentType.JSON')>",
"signature": null,
"docstring": "JSON document content."
},
"XML": {
"name": "XML",
"kind": "attribute",
"path": "omniread.pdf.parser.ContentType.XML",
"signature": "<bound method Alias.signature of Alias('XML', 'omniread.core.content.ContentType.XML')>",
"signature": null,
"docstring": "XML document content."
}
}
@@ -66,35 +59,35 @@
"name": "BaseParser",
"kind": "class",
"path": "omniread.pdf.parser.BaseParser",
"signature": "<bound method Alias.signature of Alias('BaseParser', 'omniread.core.parser.BaseParser')>",
"signature": "BaseParser(content: Content)",
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
"members": {
"supported_types": {
"name": "supported_types",
"kind": "attribute",
"path": "omniread.pdf.parser.BaseParser.supported_types",
"signature": "<bound method Alias.signature of Alias('supported_types', 'omniread.core.parser.BaseParser.supported_types')>",
"signature": null,
"docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic."
},
"content": {
"name": "content",
"kind": "attribute",
"path": "omniread.pdf.parser.BaseParser.content",
"signature": "<bound method Alias.signature of Alias('content', 'omniread.core.parser.BaseParser.content')>",
"signature": null,
"docstring": null
},
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.pdf.parser.BaseParser.parse",
"signature": "<bound method Alias.signature of Alias('parse', 'omniread.core.parser.BaseParser.parse')>",
"signature": "parse()",
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
},
"supports": {
"name": "supports",
"kind": "function",
"path": "omniread.pdf.parser.BaseParser.supports",
"signature": "<bound method Alias.signature of Alias('supports', 'omniread.core.parser.BaseParser.supports')>",
"signature": "supports()",
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
}
}
@@ -110,7 +103,7 @@
"name": "PDFParser",
"kind": "class",
"path": "omniread.pdf.parser.PDFParser",
"signature": "<bound method Class.signature of Class('PDFParser', 22, 62)>",
"signature": null,
"docstring": "Base PDF parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces PDF content-type compatibility and provides\n the extension point for implementing concrete PDF parsing strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
"members": {
"supported_types": {
@@ -124,7 +117,7 @@
"name": "parse",
"kind": "function",
"path": "omniread.pdf.parser.PDFParser.parse",
"signature": "<bound method Function.signature of Function('parse', 43, 62)>",
"signature": "parse() -> T",
"docstring": "Parse PDF content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the PDF binary payload and\n return a deterministic, structured output."
}
}

View File

@@ -4,60 +4,39 @@
"path": "omniread.pdf.scraper",
"docstring": "# Summary\n\nPDF scraping implementation for OmniRead.\n\nThis module provides a PDF-specific scraper that coordinates PDF byte\nretrieval via a client and normalizes the result into a `Content` object.\n\nThe scraper implements the core `BaseScraper` contract while delegating\nall storage and access concerns to a `BasePDFClient` implementation.",
"objects": {
"Any": {
"name": "Any",
"kind": "alias",
"path": "omniread.pdf.scraper.Any",
"signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
"docstring": null
},
"Mapping": {
"name": "Mapping",
"kind": "alias",
"path": "omniread.pdf.scraper.Mapping",
"signature": "<bound method Alias.signature of Alias('Mapping', 'typing.Mapping')>",
"docstring": null
},
"Optional": {
"name": "Optional",
"kind": "alias",
"path": "omniread.pdf.scraper.Optional",
"signature": "<bound method Alias.signature of Alias('Optional', 'typing.Optional')>",
"docstring": null
},
"Content": {
"name": "Content",
"kind": "class",
"path": "omniread.pdf.scraper.Content",
"signature": "<bound method Alias.signature of Alias('Content', 'omniread.core.content.Content')>",
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
"members": {
"raw": {
"name": "raw",
"kind": "attribute",
"path": "omniread.pdf.scraper.Content.raw",
"signature": "<bound method Alias.signature of Alias('raw', 'omniread.core.content.Content.raw')>",
"signature": null,
"docstring": "Raw content bytes as retrieved from the source."
},
"source": {
"name": "source",
"kind": "attribute",
"path": "omniread.pdf.scraper.Content.source",
"signature": "<bound method Alias.signature of Alias('source', 'omniread.core.content.Content.source')>",
"signature": null,
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
},
"content_type": {
"name": "content_type",
"kind": "attribute",
"path": "omniread.pdf.scraper.Content.content_type",
"signature": "<bound method Alias.signature of Alias('content_type', 'omniread.core.content.Content.content_type')>",
"signature": null,
"docstring": "Optional MIME type of the content, if known."
},
"metadata": {
"name": "metadata",
"kind": "attribute",
"path": "omniread.pdf.scraper.Content.metadata",
"signature": "<bound method Alias.signature of Alias('metadata', 'omniread.core.content.Content.metadata')>",
"signature": null,
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
}
}
@@ -66,35 +45,49 @@
"name": "ContentType",
"kind": "class",
"path": "omniread.pdf.scraper.ContentType",
"signature": "<bound method Alias.signature of Alias('ContentType', 'omniread.core.content.ContentType')>",
"signature": null,
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
"members": {
"HTML": {
"name": "HTML",
"kind": "attribute",
"path": "omniread.pdf.scraper.ContentType.HTML",
"signature": "<bound method Alias.signature of Alias('HTML', 'omniread.core.content.ContentType.HTML')>",
"signature": null,
"docstring": "HTML document content."
},
"PDF": {
"name": "PDF",
"kind": "attribute",
"path": "omniread.pdf.scraper.ContentType.PDF",
"signature": "<bound method Alias.signature of Alias('PDF', 'omniread.core.content.ContentType.PDF')>",
"signature": null,
"docstring": "PDF document content."
},
"XLSX": {
"name": "XLSX",
"kind": "attribute",
"path": "omniread.pdf.scraper.ContentType.XLSX",
"signature": null,
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
},
"CSV": {
"name": "CSV",
"kind": "attribute",
"path": "omniread.pdf.scraper.ContentType.CSV",
"signature": null,
"docstring": "Comma-separated-value document content."
},
"JSON": {
"name": "JSON",
"kind": "attribute",
"path": "omniread.pdf.scraper.ContentType.JSON",
"signature": "<bound method Alias.signature of Alias('JSON', 'omniread.core.content.ContentType.JSON')>",
"signature": null,
"docstring": "JSON document content."
},
"XML": {
"name": "XML",
"kind": "attribute",
"path": "omniread.pdf.scraper.ContentType.XML",
"signature": "<bound method Alias.signature of Alias('XML', 'omniread.core.content.ContentType.XML')>",
"signature": null,
"docstring": "XML document content."
}
}
@@ -103,15 +96,15 @@
"name": "BaseScraper",
"kind": "class",
"path": "omniread.pdf.scraper.BaseScraper",
"signature": "<bound method Alias.signature of Alias('BaseScraper', 'omniread.core.scraper.BaseScraper')>",
"signature": null,
"docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.pdf.scraper.BaseScraper.fetch",
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.core.scraper.BaseScraper.fetch')>",
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
"signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None)",
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object."
}
}
},
@@ -119,14 +112,14 @@
"name": "BasePDFClient",
"kind": "class",
"path": "omniread.pdf.scraper.BasePDFClient",
"signature": "<bound method Alias.signature of Alias('BasePDFClient', 'omniread.pdf.client.BasePDFClient')>",
"signature": null,
"docstring": "Abstract client responsible for retrieving PDF bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full PDF binary payload.\n - Raise retrieval-specific errors on failure.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.pdf.scraper.BasePDFClient.fetch",
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.pdf.client.BasePDFClient.fetch')>",
"signature": "fetch(source: Any)",
"docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF location, such as a file path, object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
}
}
@@ -135,15 +128,15 @@
"name": "PDFScraper",
"kind": "class",
"path": "omniread.pdf.scraper.PDFScraper",
"signature": "<bound method Class.signature of Class('PDFScraper', 20, 77)>",
"signature": "PDFScraper(*, client: BasePDFClient)",
"docstring": "Scraper for PDF sources.\n\nNotes:\n **Responsibilities:**\n\n - Delegates byte retrieval to a PDF client and normalizes output\n into `Content`.\n - Preserves caller-provided metadata.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.pdf.scraper.PDFScraper.fetch",
"signature": "<bound method Function.signature of Function('fetch', 47, 77)>",
"docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client."
"signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None) -> Content",
"docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client."
}
}
}

View File

@@ -0,0 +1,41 @@
{
"module": "omniread.xlsx.client",
"content": {
"path": "omniread.xlsx.client",
"docstring": "# Summary\n\nXLSX client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw\nOffice Open XML spreadsheet bytes from a concrete backing store.\n\nClients provide low-level access to xlsx binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems",
"objects": {
"BaseXlsxClient": {
"name": "BaseXlsxClient",
"kind": "class",
"path": "omniread.xlsx.client.BaseXlsxClient",
"signature": null,
"docstring": "Abstract client responsible for retrieving spreadsheet bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full xlsx binary payload.\n - Raise retrieval-specific errors on failure.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.xlsx.client.BaseXlsxClient.fetch",
"signature": "fetch(source: Any) -> bytes",
"docstring": "Fetch raw xlsx bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
}
}
},
"FileSystemXlsxClient": {
"name": "FileSystemXlsxClient",
"kind": "class",
"path": "omniread.xlsx.client.FileSystemXlsxClient",
"signature": null,
"docstring": "XLSX client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads spreadsheet files directly from the disk and\n returns their raw binary contents.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.xlsx.client.FileSystemXlsxClient.fetch",
"signature": "fetch(path: Path) -> bytes",
"docstring": "Read an xlsx file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the spreadsheet file.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file."
}
}
}
}
}
}

View File

@@ -0,0 +1,520 @@
{
"module": "omniread.xlsx",
"content": {
"path": "omniread.xlsx",
"docstring": "# Summary\n\nXLSX subpackage for OmniRead.\n\nProvides acquisition and parsing of Office Open XML spreadsheet (xlsx)\ncontent:\n\n- `BaseXlsxClient`: abstract backing-store client for xlsx bytes.\n- `FileSystemXlsxClient`: local filesystem implementation.\n- `XlsxScraper`: wraps fetched bytes into canonical `Content`.\n- `XlsxParserBase`: content-type-enforcing parser contract.\n- `XlsxParser`: generic string-row parser built on openpyxl.",
"objects": {
"BaseXlsxClient": {
"name": "BaseXlsxClient",
"kind": "class",
"path": "omniread.xlsx.BaseXlsxClient",
"signature": null,
"docstring": "Abstract client responsible for retrieving spreadsheet bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full xlsx binary payload.\n - Raise retrieval-specific errors on failure.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.xlsx.BaseXlsxClient.fetch",
"signature": "fetch(source: Any)",
"docstring": "Fetch raw xlsx bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
}
}
},
"FileSystemXlsxClient": {
"name": "FileSystemXlsxClient",
"kind": "class",
"path": "omniread.xlsx.FileSystemXlsxClient",
"signature": null,
"docstring": "XLSX client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads spreadsheet files directly from the disk and\n returns their raw binary contents.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.xlsx.FileSystemXlsxClient.fetch",
"signature": "fetch(path: Path)",
"docstring": "Read an xlsx file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the spreadsheet file.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file."
}
}
},
"XlsxParser": {
"name": "XlsxParser",
"kind": "class",
"path": "omniread.xlsx.XlsxParser",
"signature": "XlsxParser(content: Content, *, data_only: bool = True, read_only: bool = True)",
"docstring": "Generic xlsx parser producing string rows from a worksheet.\n\nNotes:\n **Responsibilities:**\n\n - Lazily load the workbook owned by the parser's content.\n - Normalize cells (including dates and numeric values) into\n deterministic string representations.\n - Expose sheet discovery and row extraction helpers.\n\n **Constraints:**\n\n - Cells are rendered with ``str(value)`` after trimming; date and\n datetime values are rendered in ISO format. Consumers requiring\n locale-specific formatting must convert on their side.",
"members": {
"workbook": {
"name": "workbook",
"kind": "attribute",
"path": "omniread.xlsx.XlsxParser.workbook",
"signature": null,
"docstring": "The lazily loaded workbook backing this parser's content."
},
"sheet_names": {
"name": "sheet_names",
"kind": "attribute",
"path": "omniread.xlsx.XlsxParser.sheet_names",
"signature": null,
"docstring": "Names of all worksheets contained in the workbook."
},
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.xlsx.XlsxParser.parse",
"signature": "parse()",
"docstring": "Parse the first worksheet into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the default (first) worksheet."
},
"rows": {
"name": "rows",
"kind": "function",
"path": "omniread.xlsx.XlsxParser.rows",
"signature": "rows(sheet: int | str | None = None, *, skip_empty: bool = True)",
"docstring": "Extract normalized string rows from a worksheet.\n\nArgs:\n sheet (int | str | None):\n Worksheet index or title; defaults to the first worksheet.\n skip_empty (bool):\n When True (default), rows whose cells are all blank are omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row.\n\nRaises:\n ValueError:\n If the requested sheet does not exist."
}
}
},
"XlsxParserBase": {
"name": "XlsxParserBase",
"kind": "class",
"path": "omniread.xlsx.XlsxParserBase",
"signature": null,
"docstring": "Base xlsx parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces xlsx content-type compatibility and provides\n the extension point for implementing concrete xlsx parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
"members": {
"supported_types": {
"name": "supported_types",
"kind": "attribute",
"path": "omniread.xlsx.XlsxParserBase.supported_types",
"signature": null,
"docstring": "Set of content types supported by this parser (XLSX only)."
},
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.xlsx.XlsxParserBase.parse",
"signature": "parse()",
"docstring": "Parse xlsx content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
}
}
},
"XlsxScraper": {
"name": "XlsxScraper",
"kind": "class",
"path": "omniread.xlsx.XlsxScraper",
"signature": "XlsxScraper(*, client: BaseXlsxClient)",
"docstring": "Scraper for xlsx spreadsheet documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw xlsx bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n XLSX content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.xlsx.XlsxScraper.fetch",
"signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None)",
"docstring": "Fetch an xlsx document from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw xlsx bytes, source\n identifier, XLSX content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client."
}
}
},
"client": {
"name": "client",
"kind": "module",
"path": "omniread.xlsx.client",
"signature": null,
"docstring": "# Summary\n\nXLSX client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw\nOffice Open XML spreadsheet bytes from a concrete backing store.\n\nClients provide low-level access to xlsx binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems",
"members": {
"BaseXlsxClient": {
"name": "BaseXlsxClient",
"kind": "class",
"path": "omniread.xlsx.client.BaseXlsxClient",
"signature": null,
"docstring": "Abstract client responsible for retrieving spreadsheet bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full xlsx binary payload.\n - Raise retrieval-specific errors on failure.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.xlsx.client.BaseXlsxClient.fetch",
"signature": "fetch(source: Any) -> bytes",
"docstring": "Fetch raw xlsx bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
}
}
},
"FileSystemXlsxClient": {
"name": "FileSystemXlsxClient",
"kind": "class",
"path": "omniread.xlsx.client.FileSystemXlsxClient",
"signature": null,
"docstring": "XLSX client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads spreadsheet files directly from the disk and\n returns their raw binary contents.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.xlsx.client.FileSystemXlsxClient.fetch",
"signature": "fetch(path: Path) -> bytes",
"docstring": "Read an xlsx file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the spreadsheet file.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file."
}
}
}
}
},
"parser": {
"name": "parser",
"kind": "module",
"path": "omniread.xlsx.parser",
"signature": null,
"docstring": "# Summary\n\nXLSX parser implementations for OmniRead.\n\nThis module provides a concrete, generic parser for Office Open XML\nspreadsheets. It exposes workbook sheets as lists of string rows so\ndownstream consumers can interpret tabular content without depending on\nopenpyxl directly.\n\nThe parser is intentionally statement-agnostic: it performs no header\ndetection or column interpretation beyond basic cell normalization.",
"members": {
"Content": {
"name": "Content",
"kind": "class",
"path": "omniread.xlsx.parser.Content",
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
"members": {
"raw": {
"name": "raw",
"kind": "attribute",
"path": "omniread.xlsx.parser.Content.raw",
"signature": null,
"docstring": "Raw content bytes as retrieved from the source."
},
"source": {
"name": "source",
"kind": "attribute",
"path": "omniread.xlsx.parser.Content.source",
"signature": null,
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
},
"content_type": {
"name": "content_type",
"kind": "attribute",
"path": "omniread.xlsx.parser.Content.content_type",
"signature": null,
"docstring": "Optional MIME type of the content, if known."
},
"metadata": {
"name": "metadata",
"kind": "attribute",
"path": "omniread.xlsx.parser.Content.metadata",
"signature": null,
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
}
}
},
"XlsxParserBase": {
"name": "XlsxParserBase",
"kind": "class",
"path": "omniread.xlsx.parser.XlsxParserBase",
"signature": null,
"docstring": "Base xlsx parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces xlsx content-type compatibility and provides\n the extension point for implementing concrete xlsx parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
"members": {
"supported_types": {
"name": "supported_types",
"kind": "attribute",
"path": "omniread.xlsx.parser.XlsxParserBase.supported_types",
"signature": null,
"docstring": "Set of content types supported by this parser (XLSX only)."
},
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.xlsx.parser.XlsxParserBase.parse",
"signature": "parse()",
"docstring": "Parse xlsx content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
}
}
},
"XlsxParser": {
"name": "XlsxParser",
"kind": "class",
"path": "omniread.xlsx.parser.XlsxParser",
"signature": "XlsxParser(content: Content, *, data_only: bool = True, read_only: bool = True)",
"docstring": "Generic xlsx parser producing string rows from a worksheet.\n\nNotes:\n **Responsibilities:**\n\n - Lazily load the workbook owned by the parser's content.\n - Normalize cells (including dates and numeric values) into\n deterministic string representations.\n - Expose sheet discovery and row extraction helpers.\n\n **Constraints:**\n\n - Cells are rendered with ``str(value)`` after trimming; date and\n datetime values are rendered in ISO format. Consumers requiring\n locale-specific formatting must convert on their side.",
"members": {
"workbook": {
"name": "workbook",
"kind": "attribute",
"path": "omniread.xlsx.parser.XlsxParser.workbook",
"signature": null,
"docstring": "The lazily loaded workbook backing this parser's content."
},
"sheet_names": {
"name": "sheet_names",
"kind": "attribute",
"path": "omniread.xlsx.parser.XlsxParser.sheet_names",
"signature": null,
"docstring": "Names of all worksheets contained in the workbook."
},
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.xlsx.parser.XlsxParser.parse",
"signature": "parse() -> list[list[str]]",
"docstring": "Parse the first worksheet into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the default (first) worksheet."
},
"rows": {
"name": "rows",
"kind": "function",
"path": "omniread.xlsx.parser.XlsxParser.rows",
"signature": "rows(sheet: int | str | None = None, *, skip_empty: bool = True) -> list[list[str]]",
"docstring": "Extract normalized string rows from a worksheet.\n\nArgs:\n sheet (int | str | None):\n Worksheet index or title; defaults to the first worksheet.\n skip_empty (bool):\n When True (default), rows whose cells are all blank are omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row.\n\nRaises:\n ValueError:\n If the requested sheet does not exist."
}
}
}
}
},
"parser_base": {
"name": "parser_base",
"kind": "module",
"path": "omniread.xlsx.parser_base",
"signature": null,
"docstring": "# Summary\n\nXLSX parser base implementation for OmniRead.\n\nThis module defines the **XLSX-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for Office Open\nXML spreadsheet content.",
"members": {
"ContentType": {
"name": "ContentType",
"kind": "class",
"path": "omniread.xlsx.parser_base.ContentType",
"signature": null,
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
"members": {
"HTML": {
"name": "HTML",
"kind": "attribute",
"path": "omniread.xlsx.parser_base.ContentType.HTML",
"signature": null,
"docstring": "HTML document content."
},
"PDF": {
"name": "PDF",
"kind": "attribute",
"path": "omniread.xlsx.parser_base.ContentType.PDF",
"signature": null,
"docstring": "PDF document content."
},
"XLSX": {
"name": "XLSX",
"kind": "attribute",
"path": "omniread.xlsx.parser_base.ContentType.XLSX",
"signature": null,
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
},
"CSV": {
"name": "CSV",
"kind": "attribute",
"path": "omniread.xlsx.parser_base.ContentType.CSV",
"signature": null,
"docstring": "Comma-separated-value document content."
},
"JSON": {
"name": "JSON",
"kind": "attribute",
"path": "omniread.xlsx.parser_base.ContentType.JSON",
"signature": null,
"docstring": "JSON document content."
},
"XML": {
"name": "XML",
"kind": "attribute",
"path": "omniread.xlsx.parser_base.ContentType.XML",
"signature": null,
"docstring": "XML document content."
}
}
},
"BaseParser": {
"name": "BaseParser",
"kind": "class",
"path": "omniread.xlsx.parser_base.BaseParser",
"signature": "BaseParser(content: Content)",
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
"members": {
"supported_types": {
"name": "supported_types",
"kind": "attribute",
"path": "omniread.xlsx.parser_base.BaseParser.supported_types",
"signature": null,
"docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic."
},
"content": {
"name": "content",
"kind": "attribute",
"path": "omniread.xlsx.parser_base.BaseParser.content",
"signature": null,
"docstring": null
},
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.xlsx.parser_base.BaseParser.parse",
"signature": "parse()",
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
},
"supports": {
"name": "supports",
"kind": "function",
"path": "omniread.xlsx.parser_base.BaseParser.supports",
"signature": "supports()",
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
}
}
},
"T": {
"name": "T",
"kind": "attribute",
"path": "omniread.xlsx.parser_base.T",
"signature": null,
"docstring": null
},
"XlsxParserBase": {
"name": "XlsxParserBase",
"kind": "class",
"path": "omniread.xlsx.parser_base.XlsxParserBase",
"signature": null,
"docstring": "Base xlsx parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces xlsx content-type compatibility and provides\n the extension point for implementing concrete xlsx parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
"members": {
"supported_types": {
"name": "supported_types",
"kind": "attribute",
"path": "omniread.xlsx.parser_base.XlsxParserBase.supported_types",
"signature": null,
"docstring": "Set of content types supported by this parser (XLSX only)."
},
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.xlsx.parser_base.XlsxParserBase.parse",
"signature": "parse() -> T",
"docstring": "Parse xlsx content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
}
}
}
}
},
"scraper": {
"name": "scraper",
"kind": "module",
"path": "omniread.xlsx.scraper",
"signature": null,
"docstring": "# Summary\n\nXLSX scraper for OmniRead.\n\nThis module defines the scraper responsible for acquiring raw Office Open\nXML spreadsheet content from a backing store via a configured client.\n\nThe scraper does not interpret or parse the acquired bytes; it wraps them in\nthe canonical `Content` model.",
"members": {
"Content": {
"name": "Content",
"kind": "class",
"path": "omniread.xlsx.scraper.Content",
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
"members": {
"raw": {
"name": "raw",
"kind": "attribute",
"path": "omniread.xlsx.scraper.Content.raw",
"signature": null,
"docstring": "Raw content bytes as retrieved from the source."
},
"source": {
"name": "source",
"kind": "attribute",
"path": "omniread.xlsx.scraper.Content.source",
"signature": null,
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
},
"content_type": {
"name": "content_type",
"kind": "attribute",
"path": "omniread.xlsx.scraper.Content.content_type",
"signature": null,
"docstring": "Optional MIME type of the content, if known."
},
"metadata": {
"name": "metadata",
"kind": "attribute",
"path": "omniread.xlsx.scraper.Content.metadata",
"signature": null,
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
}
}
},
"ContentType": {
"name": "ContentType",
"kind": "class",
"path": "omniread.xlsx.scraper.ContentType",
"signature": null,
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
"members": {
"HTML": {
"name": "HTML",
"kind": "attribute",
"path": "omniread.xlsx.scraper.ContentType.HTML",
"signature": null,
"docstring": "HTML document content."
},
"PDF": {
"name": "PDF",
"kind": "attribute",
"path": "omniread.xlsx.scraper.ContentType.PDF",
"signature": null,
"docstring": "PDF document content."
},
"XLSX": {
"name": "XLSX",
"kind": "attribute",
"path": "omniread.xlsx.scraper.ContentType.XLSX",
"signature": null,
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
},
"CSV": {
"name": "CSV",
"kind": "attribute",
"path": "omniread.xlsx.scraper.ContentType.CSV",
"signature": null,
"docstring": "Comma-separated-value document content."
},
"JSON": {
"name": "JSON",
"kind": "attribute",
"path": "omniread.xlsx.scraper.ContentType.JSON",
"signature": null,
"docstring": "JSON document content."
},
"XML": {
"name": "XML",
"kind": "attribute",
"path": "omniread.xlsx.scraper.ContentType.XML",
"signature": null,
"docstring": "XML document content."
}
}
},
"BaseXlsxClient": {
"name": "BaseXlsxClient",
"kind": "class",
"path": "omniread.xlsx.scraper.BaseXlsxClient",
"signature": null,
"docstring": "Abstract client responsible for retrieving spreadsheet bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full xlsx binary payload.\n - Raise retrieval-specific errors on failure.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.xlsx.scraper.BaseXlsxClient.fetch",
"signature": "fetch(source: Any)",
"docstring": "Fetch raw xlsx bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
}
}
},
"XlsxScraper": {
"name": "XlsxScraper",
"kind": "class",
"path": "omniread.xlsx.scraper.XlsxScraper",
"signature": "XlsxScraper(*, client: BaseXlsxClient)",
"docstring": "Scraper for xlsx spreadsheet documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw xlsx bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n XLSX content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.xlsx.scraper.XlsxScraper.fetch",
"signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None) -> Content",
"docstring": "Fetch an xlsx document from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw xlsx bytes, source\n identifier, XLSX content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client."
}
}
}
}
}
}
}
}

View File

@@ -0,0 +1,106 @@
{
"module": "omniread.xlsx.parser",
"content": {
"path": "omniread.xlsx.parser",
"docstring": "# Summary\n\nXLSX parser implementations for OmniRead.\n\nThis module provides a concrete, generic parser for Office Open XML\nspreadsheets. It exposes workbook sheets as lists of string rows so\ndownstream consumers can interpret tabular content without depending on\nopenpyxl directly.\n\nThe parser is intentionally statement-agnostic: it performs no header\ndetection or column interpretation beyond basic cell normalization.",
"objects": {
"Content": {
"name": "Content",
"kind": "class",
"path": "omniread.xlsx.parser.Content",
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
"members": {
"raw": {
"name": "raw",
"kind": "attribute",
"path": "omniread.xlsx.parser.Content.raw",
"signature": null,
"docstring": "Raw content bytes as retrieved from the source."
},
"source": {
"name": "source",
"kind": "attribute",
"path": "omniread.xlsx.parser.Content.source",
"signature": null,
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
},
"content_type": {
"name": "content_type",
"kind": "attribute",
"path": "omniread.xlsx.parser.Content.content_type",
"signature": null,
"docstring": "Optional MIME type of the content, if known."
},
"metadata": {
"name": "metadata",
"kind": "attribute",
"path": "omniread.xlsx.parser.Content.metadata",
"signature": null,
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
}
}
},
"XlsxParserBase": {
"name": "XlsxParserBase",
"kind": "class",
"path": "omniread.xlsx.parser.XlsxParserBase",
"signature": null,
"docstring": "Base xlsx parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces xlsx content-type compatibility and provides\n the extension point for implementing concrete xlsx parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
"members": {
"supported_types": {
"name": "supported_types",
"kind": "attribute",
"path": "omniread.xlsx.parser.XlsxParserBase.supported_types",
"signature": null,
"docstring": "Set of content types supported by this parser (XLSX only)."
},
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.xlsx.parser.XlsxParserBase.parse",
"signature": "parse()",
"docstring": "Parse xlsx content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
}
}
},
"XlsxParser": {
"name": "XlsxParser",
"kind": "class",
"path": "omniread.xlsx.parser.XlsxParser",
"signature": "XlsxParser(content: Content, *, data_only: bool = True, read_only: bool = True)",
"docstring": "Generic xlsx parser producing string rows from a worksheet.\n\nNotes:\n **Responsibilities:**\n\n - Lazily load the workbook owned by the parser's content.\n - Normalize cells (including dates and numeric values) into\n deterministic string representations.\n - Expose sheet discovery and row extraction helpers.\n\n **Constraints:**\n\n - Cells are rendered with ``str(value)`` after trimming; date and\n datetime values are rendered in ISO format. Consumers requiring\n locale-specific formatting must convert on their side.",
"members": {
"workbook": {
"name": "workbook",
"kind": "attribute",
"path": "omniread.xlsx.parser.XlsxParser.workbook",
"signature": null,
"docstring": "The lazily loaded workbook backing this parser's content."
},
"sheet_names": {
"name": "sheet_names",
"kind": "attribute",
"path": "omniread.xlsx.parser.XlsxParser.sheet_names",
"signature": null,
"docstring": "Names of all worksheets contained in the workbook."
},
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.xlsx.parser.XlsxParser.parse",
"signature": "parse() -> list[list[str]]",
"docstring": "Parse the first worksheet into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the default (first) worksheet."
},
"rows": {
"name": "rows",
"kind": "function",
"path": "omniread.xlsx.parser.XlsxParser.rows",
"signature": "rows(sheet: int | str | None = None, *, skip_empty: bool = True) -> list[list[str]]",
"docstring": "Extract normalized string rows from a worksheet.\n\nArgs:\n sheet (int | str | None):\n Worksheet index or title; defaults to the first worksheet.\n skip_empty (bool):\n When True (default), rows whose cells are all blank are omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row.\n\nRaises:\n ValueError:\n If the requested sheet does not exist."
}
}
}
}
}
}

View File

@@ -0,0 +1,127 @@
{
"module": "omniread.xlsx.parser_base",
"content": {
"path": "omniread.xlsx.parser_base",
"docstring": "# Summary\n\nXLSX parser base implementation for OmniRead.\n\nThis module defines the **XLSX-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for Office Open\nXML spreadsheet content.",
"objects": {
"ContentType": {
"name": "ContentType",
"kind": "class",
"path": "omniread.xlsx.parser_base.ContentType",
"signature": null,
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
"members": {
"HTML": {
"name": "HTML",
"kind": "attribute",
"path": "omniread.xlsx.parser_base.ContentType.HTML",
"signature": null,
"docstring": "HTML document content."
},
"PDF": {
"name": "PDF",
"kind": "attribute",
"path": "omniread.xlsx.parser_base.ContentType.PDF",
"signature": null,
"docstring": "PDF document content."
},
"XLSX": {
"name": "XLSX",
"kind": "attribute",
"path": "omniread.xlsx.parser_base.ContentType.XLSX",
"signature": null,
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
},
"CSV": {
"name": "CSV",
"kind": "attribute",
"path": "omniread.xlsx.parser_base.ContentType.CSV",
"signature": null,
"docstring": "Comma-separated-value document content."
},
"JSON": {
"name": "JSON",
"kind": "attribute",
"path": "omniread.xlsx.parser_base.ContentType.JSON",
"signature": null,
"docstring": "JSON document content."
},
"XML": {
"name": "XML",
"kind": "attribute",
"path": "omniread.xlsx.parser_base.ContentType.XML",
"signature": null,
"docstring": "XML document content."
}
}
},
"BaseParser": {
"name": "BaseParser",
"kind": "class",
"path": "omniread.xlsx.parser_base.BaseParser",
"signature": "BaseParser(content: Content)",
"docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.",
"members": {
"supported_types": {
"name": "supported_types",
"kind": "attribute",
"path": "omniread.xlsx.parser_base.BaseParser.supported_types",
"signature": null,
"docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic."
},
"content": {
"name": "content",
"kind": "attribute",
"path": "omniread.xlsx.parser_base.BaseParser.content",
"signature": null,
"docstring": null
},
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.xlsx.parser_base.BaseParser.parse",
"signature": "parse()",
"docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output."
},
"supports": {
"name": "supports",
"kind": "function",
"path": "omniread.xlsx.parser_base.BaseParser.supports",
"signature": "supports()",
"docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise."
}
}
},
"T": {
"name": "T",
"kind": "attribute",
"path": "omniread.xlsx.parser_base.T",
"signature": null,
"docstring": null
},
"XlsxParserBase": {
"name": "XlsxParserBase",
"kind": "class",
"path": "omniread.xlsx.parser_base.XlsxParserBase",
"signature": null,
"docstring": "Base xlsx parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces xlsx content-type compatibility and provides\n the extension point for implementing concrete xlsx parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.",
"members": {
"supported_types": {
"name": "supported_types",
"kind": "attribute",
"path": "omniread.xlsx.parser_base.XlsxParserBase.supported_types",
"signature": null,
"docstring": "Set of content types supported by this parser (XLSX only)."
},
"parse": {
"name": "parse",
"kind": "function",
"path": "omniread.xlsx.parser_base.XlsxParserBase.parse",
"signature": "parse() -> T",
"docstring": "Parse xlsx content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation."
}
}
}
}
}
}

View File

@@ -0,0 +1,129 @@
{
"module": "omniread.xlsx.scraper",
"content": {
"path": "omniread.xlsx.scraper",
"docstring": "# Summary\n\nXLSX scraper for OmniRead.\n\nThis module defines the scraper responsible for acquiring raw Office Open\nXML spreadsheet content from a backing store via a configured client.\n\nThe scraper does not interpret or parse the acquired bytes; it wraps them in\nthe canonical `Content` model.",
"objects": {
"Content": {
"name": "Content",
"kind": "class",
"path": "omniread.xlsx.scraper.Content",
"signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)",
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.",
"members": {
"raw": {
"name": "raw",
"kind": "attribute",
"path": "omniread.xlsx.scraper.Content.raw",
"signature": null,
"docstring": "Raw content bytes as retrieved from the source."
},
"source": {
"name": "source",
"kind": "attribute",
"path": "omniread.xlsx.scraper.Content.source",
"signature": null,
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
},
"content_type": {
"name": "content_type",
"kind": "attribute",
"path": "omniread.xlsx.scraper.Content.content_type",
"signature": null,
"docstring": "Optional MIME type of the content, if known."
},
"metadata": {
"name": "metadata",
"kind": "attribute",
"path": "omniread.xlsx.scraper.Content.metadata",
"signature": null,
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
}
}
},
"ContentType": {
"name": "ContentType",
"kind": "class",
"path": "omniread.xlsx.scraper.ContentType",
"signature": null,
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.",
"members": {
"HTML": {
"name": "HTML",
"kind": "attribute",
"path": "omniread.xlsx.scraper.ContentType.HTML",
"signature": null,
"docstring": "HTML document content."
},
"PDF": {
"name": "PDF",
"kind": "attribute",
"path": "omniread.xlsx.scraper.ContentType.PDF",
"signature": null,
"docstring": "PDF document content."
},
"XLSX": {
"name": "XLSX",
"kind": "attribute",
"path": "omniread.xlsx.scraper.ContentType.XLSX",
"signature": null,
"docstring": "Office Open XML spreadsheet (xlsx/xlsm) content."
},
"CSV": {
"name": "CSV",
"kind": "attribute",
"path": "omniread.xlsx.scraper.ContentType.CSV",
"signature": null,
"docstring": "Comma-separated-value document content."
},
"JSON": {
"name": "JSON",
"kind": "attribute",
"path": "omniread.xlsx.scraper.ContentType.JSON",
"signature": null,
"docstring": "JSON document content."
},
"XML": {
"name": "XML",
"kind": "attribute",
"path": "omniread.xlsx.scraper.ContentType.XML",
"signature": null,
"docstring": "XML document content."
}
}
},
"BaseXlsxClient": {
"name": "BaseXlsxClient",
"kind": "class",
"path": "omniread.xlsx.scraper.BaseXlsxClient",
"signature": null,
"docstring": "Abstract client responsible for retrieving spreadsheet bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full xlsx binary payload.\n - Raise retrieval-specific errors on failure.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.xlsx.scraper.BaseXlsxClient.fetch",
"signature": "fetch(source: Any)",
"docstring": "Fetch raw xlsx bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation."
}
}
},
"XlsxScraper": {
"name": "XlsxScraper",
"kind": "class",
"path": "omniread.xlsx.scraper.XlsxScraper",
"signature": "XlsxScraper(*, client: BaseXlsxClient)",
"docstring": "Scraper for xlsx spreadsheet documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw xlsx bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n XLSX content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.",
"members": {
"fetch": {
"name": "fetch",
"kind": "function",
"path": "omniread.xlsx.scraper.XlsxScraper.fetch",
"signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None) -> Content",
"docstring": "Fetch an xlsx document from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw xlsx bytes, source\n identifier, XLSX content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client."
}
}
}
}
}
}

90
docs/mcp/nav.json Normal file
View File

@@ -0,0 +1,90 @@
[
{
"module": "omniread",
"resource": "docs://modules/omniread"
},
{
"module": "omniread.core",
"resource": "docs://modules/omniread.core"
},
{
"module": "omniread.core.content",
"resource": "docs://modules/omniread.core.content"
},
{
"module": "omniread.core.parser",
"resource": "docs://modules/omniread.core.parser"
},
{
"module": "omniread.core.scraper",
"resource": "docs://modules/omniread.core.scraper"
},
{
"module": "omniread.csv",
"resource": "docs://modules/omniread.csv"
},
{
"module": "omniread.csv.client",
"resource": "docs://modules/omniread.csv.client"
},
{
"module": "omniread.csv.parser",
"resource": "docs://modules/omniread.csv.parser"
},
{
"module": "omniread.csv.parser_base",
"resource": "docs://modules/omniread.csv.parser_base"
},
{
"module": "omniread.csv.scraper",
"resource": "docs://modules/omniread.csv.scraper"
},
{
"module": "omniread.html",
"resource": "docs://modules/omniread.html"
},
{
"module": "omniread.html.parser",
"resource": "docs://modules/omniread.html.parser"
},
{
"module": "omniread.html.scraper",
"resource": "docs://modules/omniread.html.scraper"
},
{
"module": "omniread.pdf",
"resource": "docs://modules/omniread.pdf"
},
{
"module": "omniread.pdf.client",
"resource": "docs://modules/omniread.pdf.client"
},
{
"module": "omniread.pdf.parser",
"resource": "docs://modules/omniread.pdf.parser"
},
{
"module": "omniread.pdf.scraper",
"resource": "docs://modules/omniread.pdf.scraper"
},
{
"module": "omniread.xlsx",
"resource": "docs://modules/omniread.xlsx"
},
{
"module": "omniread.xlsx.client",
"resource": "docs://modules/omniread.xlsx.client"
},
{
"module": "omniread.xlsx.parser",
"resource": "docs://modules/omniread.xlsx.parser"
},
{
"module": "omniread.xlsx.parser_base",
"resource": "docs://modules/omniread.xlsx.parser_base"
},
{
"module": "omniread.xlsx.scraper",
"resource": "docs://modules/omniread.xlsx.scraper"
}
]

View File

@@ -21,33 +21,15 @@ theme:
- search.highlight
- search.share
- search.suggest
plugins:
- search
- mkdocstrings:
handlers:
python:
paths:
- .
options:
docstring_style: google
show_source: false
show_signature_annotations: true
separate_signature: true
merge_init_into_class: true
inherited_members: true
annotations_path: brief
show_root_heading: true
group_by_category: true
show_category_heading: true
show_object_full_path: false
show_symbol_type_heading: true
icon:
logo: material/book-open-page-variant
repo: fontawesome/brands/github
markdown_extensions:
- pymdownx.superfences
- pymdownx.inlinehilite
- pymdownx.snippets
- admonition
- pymdownx.details
- pymdownx.superfences
- pymdownx.highlight:
linenums: true
anchor_linenums: true
@@ -62,7 +44,31 @@ markdown_extensions:
- pymdownx.caret
- pymdownx.tilde
- pymdownx.mark
extra_css:
- https://unpkg.com/dracula-prism/dist/css/dracula-prism.css
plugins:
- search
- mkdocstrings:
handlers:
python:
paths:
- ..
options:
docstring_style: google
show_source: false
show_signature_annotations: true
separate_signature: true
merge_init_into_class: true
inherited_members: true
annotations_path: brief
show_root_heading: true
group_by_category: true
show_category_heading: true
show_object_full_path: false
show_symbol_type_heading: true
site_name: omniread
docs_dir: lib
site_dir: ../site/lib
nav:
- Home: index.md
- Core API:
@@ -79,3 +85,15 @@ nav:
- pdf/client.md
- pdf/parser.md
- pdf/scraper.md
- CSV Handling:
- csv/index.md
- csv/client.md
- csv/parser_base.md
- csv/parser.md
- csv/scraper.md
- XLSX Handling:
- xlsx/index.md
- xlsx/client.md
- xlsx/parser_base.md
- xlsx/parser.md
- xlsx/scraper.md

69
docs/mkdocs.wiki.yml Normal file
View File

@@ -0,0 +1,69 @@
site_name: OmniRead Documentation
docs_dir: wiki
site_dir: ../site/wiki
nav:
- Home: index.md
- Overview: 01_overview.md
- How to Use: 02_how_to_use.md
- Extending OmniRead: 03_extending.md
- Development: 04_development.md
theme:
name: material
palette:
scheme: slate
primary: blue grey
accent: teal
font:
text: Roboto
code: JetBrains Mono
features:
- navigation.sections
- navigation.expand
- navigation.top
- navigation.instant
- navigation.tracking
- navigation.indexes
- content.code.copy
- content.code.annotate
- content.tabs.link
- content.action.edit
- search.highlight
- search.share
- search.suggest
- navigation.tabs
- toc.integrate
- header.autohide
- announce.dismiss
- footer.social
- content.code.select
- content.code.line_numbers
- content.tooltips
icon:
logo: material/book-open-page-variant
repo: fontawesome/brands/github
markdown_extensions:
- pymdownx.superfences
- pymdownx.inlinehilite
- pymdownx.snippets
- admonition
- pymdownx.details
- pymdownx.highlight:
linenums: true
anchor_linenums: true
line_spans: __span
pygments_lang_class: true
- pymdownx.tabbed:
alternate_style: true
- pymdownx.tasklist:
custom_checkbox: true
- tables
- footnotes
- pymdownx.caret
- pymdownx.tilde
- pymdownx.mark
extra_css:
- https://unpkg.com/dracula-prism/dist/css/dracula-prism.css
plugins:
- search

86
docs/wiki/01_overview.md Normal file
View File

@@ -0,0 +1,86 @@
# 🧱 Overview
OmniRead is designed as a **decoupled content engine** with three distinct
layers. Understanding them is the key to using and extending the library.
---
## 🏗️ Architecture
```text
┌─────────────────────────────┐
│ Source (URL, file, storage) │
└────────────┬────────────────┘
│
┌─────────▼──────────┐
│ Scraper / Client │ fetches raw bytes
└─────────┬──────────┘
│ returns
┌─────────▼──────────┐
│ Content │ raw + source + type
└─────────┬──────────┘
│
┌─────────▼──────────┐
│ Parser │ parse() → structured T
└────────────────────┘
```
1. **Scraper** (`BaseScraper`) — fetches raw bytes from a source (HTTP URL,
filesystem path, object storage). Returns a `Content` instance.
Scrapers never interpret content.
2. **Content** (`Content`) — the canonical exchange model: raw bytes,
source identifier, and optional `ContentType` enum.
3. **Parser** (`BaseParser[T]`) — receives a `Content` and returns a
structured result of type `T` via `parse()`.
---
## 📦 The `Content` model
Defined in `omniread.core.content`:
```python
from dataclasses import dataclass
from omniread import Content, ContentType
@dataclass(slots=True)
class Content:
raw: bytes
source: str
content_type: ContentType | None = None
```
- `raw` — the raw bytes exactly as retrieved.
- `source` — URL, file path, or logical name identifying the origin.
- `content_type` — optional `ContentType` enum value.
---
## 🎭 The `ContentType` enum
| Value | MIME | Used by |
|---|---|---|
| `HTML` | `text/html` | HTMLScraper |
| `PDF` | `application/pdf` | PDFScraper |
| `XLSX` | `application/vnd.openxmlformats-...` | XlsxScraper |
| `CSV` | `text/csv` | CsvScraper |
| `JSON` | `application/json` | — |
| `XML` | `application/xml` | — |
---
## 🧩 Format modules at a glance
| Module | Scraper | Parser | Client | Notes |
|---|---|---|---|---|
| `omniread.html` | `HTMLScraper` | `HTMLParser` | — | httpx + BeautifulSoup |
| `omniread.pdf` | `PDFScraper` | `PDFParser` | `FileSystemPDFClient` | explicit client layer |
| `omniread.csv` | `CsvScraper` | `CsvParser` | `FileSystemCsvClient` | stdlib csv module |
| `omniread.xlsx` | `XlsxScraper` | `XlsxParser` | `FileSystemXlsxClient` | openpyxl-backed |
---
## 📚 Read Next
- [How to Use](02_how_to_use.md) — working examples per format.
- [Extending OmniRead](03_extending.md) — subclassing scrapers and parsers.

109
docs/wiki/02_how_to_use.md Normal file
View File

@@ -0,0 +1,109 @@
# 🖥️ How to Use
Every format follows the same pipeline: **scrape → `Content` → parse**.
This page shows each supported format with working patterns.
---
## 🌐 HTML
```python
from omniread import HTMLScraper, HTMLParser
class TitleParser(HTMLParser[str]):
def parse(self) -> str:
return self._soup.title.string
scraper = HTMLScraper() # httpx under the hood
content = scraper.fetch("https://example.com")
title = TitleParser(content).parse()
```
`HTMLScraper` accepts an optional `client` (an `httpx.Client`) for transport
control — the test suite wires one to a mock transport.
---
## 📕 PDF
PDFs need a **client** to supply raw bytes before parsing:
```python
from pathlib import Path
from omniread import FileSystemPDFClient, PDFScraper, PDFParser
class TextPDFParser(PDFParser[str]):
def parse(self) -> str:
# implement your extraction logic
return self.content.raw # bytes, decode as needed
client = FileSystemPDFClient()
scraper = PDFScraper(client=client)
content = scraper.fetch(Path("document.pdf"))
result = TextPDFParser(content).parse()
```
`PDFParser` subclasses receive `self.content` and implement `parse()`.
---
## 📊 CSV
```python
from omniread import FileSystemCsvClient, CsvScraper, CsvParser
scraper = CsvScraper(client=FileSystemCsvClient())
content = scraper.fetch("data.csv")
parser = CsvParser(content)
for row in parser.rows():
print(row)
```
`CsvParser.rows()` yields string rows trimmed of empties by default
(`skip_empty=True`).
---
## 📑 XLSX
```python
from omniread import FileSystemXlsxClient, XlsxScraper, XlsxParser
scraper = XlsxScraper(client=FileSystemXlsxClient())
content = scraper.fetch("statement.xlsx")
parser = XlsxParser(content)
print(parser.sheet_names) # e.g. ["Statement"]
rows = parser.rows(sheet="Statement") # by name or index
all_rows = parser.parse() # alias for rows()
```
Key behaviors:
- `rows(skip_empty=False)` keeps blank rows (off by default).
- Cells render as trimmed strings; date cells convert to ISO format
(`2026-06-01T00:00:00`).
- `rows(sheet="Missing")` raises for an unknown sheet.
---
## 🔀 End-to-end flow
```python
content = scraper.fetch(source) # 1. acquire → Content
assert isinstance(content.raw, bytes)
assert content.content_type is not None
parser = MyParser(content) # 2. interpret → T
result = parser.parse()
```
---
## 📚 Read Next
- [Overview](01_overview.md) — the full architecture.
- [Extending OmniRead](03_extending.md) — custom clients and parsers.

104
docs/wiki/03_extending.md Normal file
View File

@@ -0,0 +1,104 @@
# 🧩 Extending OmniRead
OmniRead is meant to be extended by subclassing. All public extension points
are generic over their result type, so your parser returns exactly the shape
you need.
---
## 🧬 Custom parsers
Subclass `BaseParser[T]` (or a format parser) and implement `parse()`:
```python
from pydantic import BaseModel
from omniread import HTMLParser
class Page(BaseModel):
title: str
content: str | None
class PageParser(HTMLParser[Page]):
def parse(self) -> Page:
soup = self._soup
div = soup.find("div", id="content")
return Page(
title=soup.title.string,
content=div.get_text() if div else None,
)
```
The parsed page is validated by Pydantic on construction — no manual
assertions required.
---
## 🧬 Custom PDF parsers
PDF binary layout is format-specific, so parsers return your own model:
```python
from typing import Literal
from pydantic import BaseModel
from omniread import PDFParser
class ParsedPDF(BaseModel):
size_bytes: int
magic: Literal[b"%PDF"]
class SimplePDFParser(PDFParser[ParsedPDF]):
def parse(self) -> ParsedPDF:
if not self.content.raw.startswith(b"%PDF"):
raise ValueError("Not a valid PDF")
return ParsedPDF(size_bytes=len(self.content.raw), magic=b"%PDF")
```
---
## 🧬 Custom clients
Clients supply raw bytes to a scraper. For PDFs, subclass
`BasePDFClient` (or `FileSystemPDFClient`) and implement
`fetch(source) -> bytes`:
```python
from omniread.pdf.client import BasePDFClient
class MockPDFClient(BasePDFClient):
def fetch(self, source):
return b"%PDF ..." # bytes for the logical identifier
```
The same pattern applies to `BaseCsvClient` and `BaseXlsxClient`.
---
## 🚀 Custom scrapers
Festch something that a built-in scraper does not cover by extending
`BaseScraper`:
```python
from omniread import BaseScraper, Content, ContentType
class StorageScraper(BaseScraper):
def fetch(self, source, *, metadata=None):
raw = my_object_storage.download(source) # your I/O
return Content(raw=raw, source=source, content_type=ContentType.JSON)
```
---
## ✅ Extension checklist
1. Keep **scraper** and **parser** separate — never mix I/O into `parse()`.
2. Return `Content` from any scraper/client so downstream stays uniform.
3. Return a *typed* result from your parser (Pydantic model, dataclass, str).
4. Test your custom layers with a mock client, not a live network.
---
## 📚 Read Next
- [How to Use](02_how_to_use.md) — built-in example flows.
- [Development](04_development.md) — running tests and docs.

View File

@@ -0,0 +1,87 @@
# 🛠️ Development
Working on `omniread` itself.
---
## 📂 Repository layout
| Path | Purpose |
|---|---|
| `omniread/` | The library package (core + html, pdf, csv, xlsx modules) |
| `omniread/*.pyi` | Type stubs kept in sync with implementations |
| `tests/` | End-to-end and unit tests against mock transports/clients |
| `covers` | — |
| `docs/lib/` | Generated library reference (docforge, flat layout) |
| `docs/mcp/` | Machine-readable bundle served by the MCP server |
| `docs/wiki/` | This hand-written wiki |
---
## 🔧 Setup
```bash
python -m venv .venv
.venv/Scripts/pip install -e ".[dev]"
```
---
## 🧪 Tests
Run the suite (offline; a mock httpx transport and mock PDF client are used):
```bash
.venv/Scripts/pytest
```
Coverage spans end-to-end scrape → parse flows for HTML, PDF, and XLSX,
plus client validation and CSV/XLSX parsing edge cases.
---
## ✅ Quality gates
The CI quality gate runs, matching the Drone pipeline:
```bash
.venv/Scripts/black --check .
.venv/Scripts/ruff check .
.venv/Scripts/mypy
.venv/Scripts/pytest
```
---
## 📝 Building documentation (docforge)
The site is generated by [`docforge`](https://git.aetoskia.com/aetos/doc-forge)
and served per kind under `site/{kind}`:
```bash
doc-forge build \
--mkdocs --mcp \
--module-is-source --module omniread \
--site-name "OmniRead"
```
- `--module-is-source` renders the flat `docs/lib/` layout (no nesting under
`omniread/`), matching `docforge.nav.yml` and `docs/mkdocs.lib.yml`.
- `--mcp` regenerates the structured bundle in `docs/mcp/`.
- `--wiki` builds this wiki.
- `gen_api.py` (if present) regenerates the API docs.
Preview locally:
```bash
doc-forge serve --lib
doc-forge serve --wiki
doc-forge serve --mcp
```
---
## 📚 Read Next
- [Extending OmniRead](03_extending.md) — custom parsers, scrapers, clients.
- [Overview](01_overview.md) — the core architecture.

64
docs/wiki/index.md Normal file
View File

@@ -0,0 +1,64 @@
# 📄 OmniRead — Format-Agnostic Content Acquisition and Parsing
`OmniRead` is a composable content ingestion framework with pluggable scrapers
and parsers for HTML, PDF, CSV, and XLSX. It cleanly separates fetching
(I/O) from interpretation (parsing) through a canonical `Content` exchange
model, giving each format its own scraper and parser while sharing one
uniform pipeline.
> **Doc model:** this wiki is written for humans — how‑to guides, examples,
> and extension recipes. The authoritative API contracts live in the code
> (GSDFC docstrings) and the machine‑readable bundle under `docs/mcp/`.
---
## 🚀 Key Features
* 🧱 **Content/Scraper/Parser architecture** — fetch and interpret in
independent, testable layers
* 📄 **Canonical `Content` model** — raw bytes + metadata passed between all
layers via one dataclass
* 🌐 **HTML** — HTTP-based scraper with httpx; DOM parsing via BeautifulSoup
* 📕 **PDF** — client + scraper for raw PDF bytes; user-defined parsers
* 📊 **CSV** — standard-library CSV parser over `rows()` iterator
* 📑 **XLSX** — openpyxl-backed spreadsheet parser with sheet selection,
empty-row control, and ISO date conversion
---
## ⚡ Quick Start
```python
from omniread import HTMLScraper, HTMLParser
class TitleParser(HTMLParser[str]):
def parse(self) -> str:
return self._soup.title.string
content = HTMLScraper().fetch("https://example.com")
title = TitleParser(content).parse()
assert title == "Example Domain"
```
---
## 📁 Documentation Structure
| Section | What you'll find |
|---|---|
| [Overview](01_overview.md) | The three-layer architecture and the `Content` model |
| [How to Use](02_how_to_use.md) | Ready-to-run examples for HTML, PDF, CSV, and XLSX |
| [Extending OmniRead](03_extending.md) | Subclassing scrapers, parsers, and clients |
| [Development](04_development.md) | Setup, tests, and regenerating docs |
---
## 🔗 Related Resources
* **Source Code:** [Gitea Repository](https://git.aetoskia.com/aetos/omniread)
* **Internal PyPI:** [pip.aetoskia.com/simple/omniread](https://pip.aetoskia.com/simple/omniread)
* **CI:** Builds and publishes tagged releases, gated on black / ruff / mypy / pytest.
---
© Aetoskia Internal — `omniread` 0.0.1

File diff suppressed because it is too large Load Diff

View File

@@ -1,50 +0,0 @@
[
{
"module": "omniread",
"resource": "doc://modules/omniread"
},
{
"module": "omniread.core",
"resource": "doc://modules/omniread.core"
},
{
"module": "omniread.core.content",
"resource": "doc://modules/omniread.core.content"
},
{
"module": "omniread.core.parser",
"resource": "doc://modules/omniread.core.parser"
},
{
"module": "omniread.core.scraper",
"resource": "doc://modules/omniread.core.scraper"
},
{
"module": "omniread.html",
"resource": "doc://modules/omniread.html"
},
{
"module": "omniread.html.parser",
"resource": "doc://modules/omniread.html.parser"
},
{
"module": "omniread.html.scraper",
"resource": "doc://modules/omniread.html.scraper"
},
{
"module": "omniread.pdf",
"resource": "doc://modules/omniread.pdf"
},
{
"module": "omniread.pdf.client",
"resource": "doc://modules/omniread.pdf.client"
},
{
"module": "omniread.pdf.parser",
"resource": "doc://modules/omniread.pdf.parser"
},
{
"module": "omniread.pdf.scraper",
"resource": "doc://modules/omniread.pdf.scraper"
}
]

View File

@@ -132,23 +132,19 @@ __all__ = [
# core
"Content",
"ContentType",
# html
"HTMLScraper",
"HTMLParser",
# pdf
"FileSystemPDFClient",
"PDFScraper",
"PDFParser",
# csv
"BaseCsvClient",
"FileSystemCsvClient",
"CsvScraper",
"CsvParser",
"CsvParserBase",
# xlsx
"BaseXlsxClient",
"FileSystemXlsxClient",

View File

@@ -1,6 +1,6 @@
from .core import Content, ContentType
from .html import HTMLScraper, HTMLParser
from .pdf import FileSystemPDFClient, PDFScraper, PDFParser
from .html import HTMLParser, HTMLScraper
from .pdf import FileSystemPDFClient, PDFParser, PDFScraper
__all__ = [
"Content",

View File

@@ -11,9 +11,10 @@ retrieved or parsed. Format-specific behavior and metadata must not alter
the semantic meaning of these models.
"""
from enum import Enum
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Any, Mapping, Optional
from enum import Enum
from typing import Any
class ContentType(str, Enum):
@@ -72,12 +73,12 @@ class Content:
Identifier of the content origin (URL, file path, or logical name).
"""
content_type: Optional[ContentType] = None
content_type: ContentType | None = None
"""
Optional MIME type of the content, if known.
"""
metadata: Optional[Mapping[str, Any]] = None
metadata: Mapping[str, Any] | None = None
"""
Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes).
"""

View File

@@ -1,15 +1,24 @@
from collections.abc import Mapping
from enum import Enum
from typing import Any, Mapping, Optional
from typing import Any
class ContentType(str, Enum):
HTML = "text/html"
PDF = "application/pdf"
XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
CSV = "text/csv"
JSON = "application/json"
XML = "application/xml"
class Content:
raw: bytes
source: str
content_type: Optional[ContentType]
metadata: Optional[Mapping[str, Any]]
def __init__(self, raw: bytes, source: str, content_type: Optional[ContentType] = ..., metadata: Optional[Mapping[str, Any]] = ...) -> None: ...
content_type: ContentType | None
metadata: Mapping[str, Any] | None
def __init__(
self,
raw: bytes,
source: str,
content_type: ContentType | None = ...,
metadata: Mapping[str, Any] | None = ...,
) -> None: ...

View File

@@ -20,7 +20,7 @@ Parsers are not responsible for:
"""
from abc import ABC, abstractmethod
from typing import Generic, TypeVar, Set
from typing import Generic, TypeVar
from .content import Content, ContentType
@@ -46,7 +46,7 @@ class BaseParser(ABC, Generic[T]):
- Implementations must remain deterministic for a given input.
"""
supported_types: Set[ContentType] = set()
supported_types: set[ContentType] = set()
"""
Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic.
"""

View File

@@ -1,11 +1,12 @@
from abc import ABC, abstractmethod
from typing import Generic, TypeVar, Set
from typing import Generic, TypeVar
from .content import Content, ContentType
T = TypeVar("T")
class BaseParser(ABC, Generic[T]):
supported_types: Set[ContentType]
supported_types: set[ContentType]
content: Content
def __init__(self, content: Content) -> None: ...
@abstractmethod

View File

@@ -22,7 +22,8 @@ All interpretation must be delegated to parsers.
"""
from abc import ABC, abstractmethod
from typing import Any, Mapping, Optional
from collections.abc import Mapping
from typing import Any
from .content import Content
@@ -53,7 +54,7 @@ class BaseScraper(ABC):
self,
source: str,
*,
metadata: Optional[Mapping[str, Any]] = None,
metadata: Mapping[str, Any] | None = None,
) -> Content:
"""
Fetch raw content from the given source.
@@ -62,7 +63,7 @@ class BaseScraper(ABC):
source (str):
Location identifier (URL, file path, S3 URI, etc.).
metadata (Optional[Mapping[str, Any]], optional):
metadata (Mapping[str, Any] | None, optional):
Optional hints for the scraper (headers, auth, etc.).
Returns:

View File

@@ -1,7 +1,11 @@
from abc import ABC, abstractmethod
from typing import Any, Mapping, Optional
from collections.abc import Mapping
from typing import Any
from .content import Content
class BaseScraper(ABC):
@abstractmethod
def fetch(self, source: str, *, metadata: Optional[Mapping[str, Any]] = ...) -> Content: ...
def fetch(
self, source: str, *, metadata: Mapping[str, Any] | None = ...
) -> Content: ...

12
omniread/csv/__init__.pyi Normal file
View File

@@ -0,0 +1,12 @@
from .client import BaseCsvClient, FileSystemCsvClient
from .parser import CsvParser
from .parser_base import CsvParserBase
from .scraper import CsvScraper
__all__ = [
"BaseCsvClient",
"FileSystemCsvClient",
"CsvScraper",
"CsvParser",
"CsvParserBase",
]

View File

@@ -17,9 +17,9 @@ Typical backing stores include:
- Network file systems
"""
from typing import Any
from abc import ABC, abstractmethod
from pathlib import Path
from typing import Any
class BaseCsvClient(ABC):

10
omniread/csv/client.pyi Normal file
View File

@@ -0,0 +1,10 @@
from abc import ABC, abstractmethod
from pathlib import Path
from typing import Any
class BaseCsvClient(ABC):
@abstractmethod
def fetch(self, source: Any) -> bytes: ...
class FileSystemCsvClient(BaseCsvClient):
def fetch(self, path: Path) -> bytes: ...

View File

@@ -13,15 +13,15 @@ detection or column interpretation beyond basic cell normalization and
delimiter detection.
"""
from io import StringIO
from csv import Sniffer, reader
from typing import List
from io import StringIO
from omniread.core.content import Content
from .parser_base import CsvParserBase
class CsvParser(CsvParserBase):
class CsvParser(CsvParserBase[list[list[str]]]):
"""
Generic csv parser producing string rows from the document.
@@ -54,17 +54,17 @@ class CsvParser(CsvParserBase):
"""
super().__init__(content)
def parse(self) -> List[List[str]]:
def parse(self) -> list[list[str]]:
"""
Parse the document into normalized string rows.
Returns:
List[List[str]]:
list[list[str]]:
Rows of the document.
"""
return self.rows()
def rows(self, *, skip_empty: bool = True) -> List[List[str]]:
def rows(self, *, skip_empty: bool = True) -> list[list[str]]:
"""
Extract normalized string rows from the document.
@@ -74,7 +74,7 @@ class CsvParser(CsvParserBase):
omitted.
Returns:
List[List[str]]:
list[list[str]]:
Normalized rows; trailing blank cells are trimmed per row.
"""
text = self._decode(self.content.raw)
@@ -84,7 +84,7 @@ class CsvParser(CsvParserBase):
delimiter = dialect.delimiter
except Exception:
delimiter = ","
out: List[List[str]] = []
out: list[list[str]] = []
for row in reader(StringIO(text), delimiter=delimiter):
cells = [c.strip() for c in row]
while cells and not cells[-1]:

8
omniread/csv/parser.pyi Normal file
View File

@@ -0,0 +1,8 @@
from omniread.core.content import Content
from .parser_base import CsvParserBase
class CsvParser(CsvParserBase[list[list[str]]]):
def __init__(self, content: Content) -> None: ...
def parse(self) -> list[list[str]]: ...
def rows(self, *, skip_empty: bool = True) -> list[list[str]]: ...

View File

@@ -8,8 +8,8 @@ format-agnostic `BaseParser` with constraints appropriate for
comma-separated-value documents.
"""
from typing import Generic, TypeVar
from abc import abstractmethod
from typing import Generic, TypeVar
from omniread.core.content import ContentType
from omniread.core.parser import BaseParser

View File

@@ -0,0 +1,13 @@
from abc import abstractmethod
from typing import Generic, TypeVar
from omniread.core.content import ContentType
from omniread.core.parser import BaseParser
T = TypeVar("T")
class CsvParserBase(BaseParser[T], Generic[T]):
supported_types: set[ContentType]
@abstractmethod
def parse(self) -> T: ...

View File

@@ -11,9 +11,11 @@ The scraper does not interpret or parse the acquired bytes; it wraps them in
the canonical `Content` model.
"""
from typing import Any, Mapping, Optional
from collections.abc import Mapping
from typing import Any
from omniread.core.content import Content, ContentType
from .client import BaseCsvClient
@@ -48,7 +50,7 @@ class CsvScraper:
self,
source: Any,
*,
metadata: Optional[Mapping[str, Any]] = None,
metadata: Mapping[str, Any] | None = None,
) -> Content:
"""
Fetch a csv document from the given source.
@@ -57,7 +59,7 @@ class CsvScraper:
source (Any):
Identifier of the csv source as understood by the
configured client.
metadata (Optional[Mapping[str, Any]], optional):
metadata (Mapping[str, Any] | None, optional):
Optional metadata to attach to the returned content.
Returns:

12
omniread/csv/scraper.pyi Normal file
View File

@@ -0,0 +1,12 @@
from collections.abc import Mapping
from typing import Any
from omniread.core.content import Content
from .client import BaseCsvClient
class CsvScraper:
def __init__(self, *, client: BaseCsvClient) -> None: ...
def fetch(
self, source: Any, *, metadata: Mapping[str, Any] | None = ...
) -> Content: ...

View File

@@ -30,7 +30,6 @@ use this package only when HTML-specific behavior is required.
---
"""
from .scraper import HTMLScraper
from .parser import HTMLParser

View File

@@ -1,4 +1,4 @@
from .scraper import HTMLScraper
from .parser import HTMLParser
from .scraper import HTMLScraper
__all__ = ["HTMLScraper", "HTMLParser"]

View File

@@ -16,12 +16,12 @@ Concrete parsers must subclass `HTMLParser` and implement the `parse()` method
to return a structured representation appropriate for their use case.
"""
from typing import Any, Generic, TypeVar, Optional
from abc import abstractmethod
from typing import Any, Generic, TypeVar
from bs4 import BeautifulSoup, Tag
from omniread.core.content import ContentType, Content
from omniread.core.content import Content, ContentType
from omniread.core.parser import BaseParser
T = TypeVar("T")
@@ -117,7 +117,7 @@ class HTMLParser(BaseParser[T], Generic[T]):
return div.get_text(separator=separator, strip=True)
@staticmethod
def parse_link(a: Tag) -> Optional[str]:
def parse_link(a: Tag) -> str | None:
"""
Extract the hyperlink reference from an `<a>` element.
@@ -126,7 +126,7 @@ class HTMLParser(BaseParser[T], Generic[T]):
BeautifulSoup tag representing an anchor.
Returns:
Optional[str]:
str | None:
The value of the `href` attribute, or None if absent.
"""
return a.get("href")
@@ -146,10 +146,7 @@ class HTMLParser(BaseParser[T], Generic[T]):
"""
rows: list[list[str]] = []
for tr in table.find_all("tr"):
cells = [
cell.get_text(strip=True)
for cell in tr.find_all(["td", "th"])
]
cells = [cell.get_text(strip=True) for cell in tr.find_all(["td", "th"])]
if cells:
rows.append(cells)
return rows

View File

@@ -1,6 +1,8 @@
from typing import Any, Generic, TypeVar, Optional, list, dict
from bs4 import BeautifulSoup, Tag
from omniread.core.content import ContentType, Content
from typing import Any, Generic, TypeVar
from bs4 import Tag
from omniread.core.content import Content, ContentType
from omniread.core.parser import BaseParser
T = TypeVar("T")
@@ -12,7 +14,7 @@ class HTMLParser(BaseParser[T], Generic[T]):
@staticmethod
def parse_div(div: Tag, *, separator: str = ...) -> str: ...
@staticmethod
def parse_link(a: Tag) -> Optional[str]: ...
def parse_link(a: Tag) -> str | None: ...
@staticmethod
def parse_table(table: Tag) -> list[list[str]]: ...
def parse_meta(self) -> dict[str, Any]: ...

View File

@@ -20,8 +20,10 @@ This scraper is not responsible for:
- Managing crawl policies or rate limiting
"""
from collections.abc import Mapping
from typing import Any
import httpx
from typing import Any, Mapping, Optional
from omniread.core.content import Content, ContentType
from omniread.core.scraper import BaseScraper
@@ -51,7 +53,7 @@ class HTMLScraper(BaseScraper):
*,
client: httpx.Client | None = None,
timeout: float = 15.0,
headers: Optional[Mapping[str, str]] = None,
headers: Mapping[str, str] | None = None,
follow_redirects: bool = True,
):
"""
@@ -62,7 +64,7 @@ class HTMLScraper(BaseScraper):
Optional pre-configured `httpx.Client`. If omitted, a client is created internally.
timeout (float, optional):
Request timeout in seconds.
headers (Optional[Mapping[str, str]], optional):
headers (Mapping[str, str] | None, optional):
Optional default HTTP headers.
follow_redirects (bool, optional):
Whether to follow HTTP redirects.
@@ -97,15 +99,13 @@ class HTMLScraper(BaseScraper):
base_ct = raw_ct.split(";", 1)[0].strip().lower()
if base_ct != self.content_type.value:
raise ValueError(
f"Expected HTML content, got '{raw_ct}'"
)
raise ValueError(f"Expected HTML content, got '{raw_ct}'")
def fetch(
self,
source: str,
*,
metadata: Optional[Mapping[str, Any]] = None,
metadata: Mapping[str, Any] | None = None,
) -> Content:
"""
Fetch an HTML document from the given source.
@@ -113,7 +113,7 @@ class HTMLScraper(BaseScraper):
Args:
source (str):
URL of the HTML document.
metadata (Optional[Mapping[str, Any]], optional):
metadata (Mapping[str, Any] | None, optional):
Optional metadata to be merged into the returned content.
Returns:

View File

@@ -1,10 +1,22 @@
from collections.abc import Mapping
from typing import Any
import httpx
from typing import Any, Mapping, Optional
from omniread.core.content import Content, ContentType
from omniread.core.scraper import BaseScraper
class HTMLScraper(BaseScraper):
content_type: ContentType
def __init__(self, *, client: Optional[httpx.Client] = ..., timeout: float = ..., headers: Optional[Mapping[str, str]] = ..., follow_redirects: bool = ...) -> None: ...
def __init__(
self,
*,
client: httpx.Client | None = ...,
timeout: float = ...,
headers: Mapping[str, str] | None = ...,
follow_redirects: bool = ...,
) -> None: ...
def validate_content_type(self, response: httpx.Response) -> None: ...
def fetch(self, source: str, *, metadata: Optional[Mapping[str, Any]] = ...) -> Content: ...
def fetch(
self, source: str, *, metadata: Mapping[str, Any] | None = ...
) -> Content: ...

View File

@@ -1,5 +1,5 @@
from .client import FileSystemPDFClient
from .scraper import PDFScraper
from .parser import PDFParser
from .scraper import PDFScraper
__all__ = ["FileSystemPDFClient", "PDFScraper", "PDFParser"]

View File

@@ -17,9 +17,9 @@ Typical backing stores include:
- Network file systems
"""
from typing import Any
from abc import ABC, abstractmethod
from pathlib import Path
from typing import Any
class BasePDFClient(ABC):

View File

@@ -10,8 +10,8 @@ PDF parsers are responsible for interpreting binary PDF data and producing
structured representations suitable for downstream consumption.
"""
from typing import Generic, TypeVar
from abc import abstractmethod
from typing import Generic, TypeVar
from omniread.core.content import ContentType
from omniread.core.parser import BaseParser

View File

@@ -1,5 +1,6 @@
from abc import abstractmethod
from typing import Generic, TypeVar
from omniread.core.content import ContentType
from omniread.core.parser import BaseParser

View File

@@ -10,7 +10,8 @@ The scraper implements the core `BaseScraper` contract while delegating
all storage and access concerns to a `BasePDFClient` implementation.
"""
from typing import Any, Mapping, Optional
from collections.abc import Mapping
from typing import Any
from omniread.core.content import Content, ContentType
from omniread.core.scraper import BaseScraper
@@ -48,7 +49,7 @@ class PDFScraper(BaseScraper):
self,
source: Any,
*,
metadata: Optional[Mapping[str, Any]] = None,
metadata: Mapping[str, Any] | None = None,
) -> Content:
"""
Fetch a PDF document from the given source.
@@ -56,7 +57,7 @@ class PDFScraper(BaseScraper):
Args:
source (Any):
Identifier of the PDF source as understood by the configured PDF client.
metadata (Optional[Mapping[str, Any]], optional):
metadata (Mapping[str, Any] | None, optional):
Optional metadata to attach to the returned content.
Returns:

View File

@@ -1,8 +1,13 @@
from typing import Any, Mapping, Optional
from omniread.core.content import Content, ContentType
from collections.abc import Mapping
from typing import Any
from omniread.core.content import Content
from omniread.core.scraper import BaseScraper
from .client import BasePDFClient
class PDFScraper(BaseScraper):
def __init__(self, *, client: BasePDFClient) -> None: ...
def fetch(self, source: Any, *, metadata: Optional[Mapping[str, Any]] = ...) -> Content: ...
def fetch(
self, source: Any, *, metadata: Mapping[str, Any] | None = ...
) -> Content: ...

0
omniread/py.typed Normal file
View File

View File

@@ -0,0 +1,12 @@
from .client import BaseXlsxClient, FileSystemXlsxClient
from .parser import XlsxParser
from .parser_base import XlsxParserBase
from .scraper import XlsxScraper
__all__ = [
"BaseXlsxClient",
"FileSystemXlsxClient",
"XlsxScraper",
"XlsxParser",
"XlsxParserBase",
]

View File

@@ -17,9 +17,9 @@ Typical backing stores include:
- Network file systems
"""
from typing import Any
from abc import ABC, abstractmethod
from pathlib import Path
from typing import Any
class BaseXlsxClient(ABC):

10
omniread/xlsx/client.pyi Normal file
View File

@@ -0,0 +1,10 @@
from abc import ABC, abstractmethod
from pathlib import Path
from typing import Any
class BaseXlsxClient(ABC):
@abstractmethod
def fetch(self, source: Any) -> bytes: ...
class FileSystemXlsxClient(BaseXlsxClient):
def fetch(self, path: Path) -> bytes: ...

View File

@@ -14,15 +14,16 @@ detection or column interpretation beyond basic cell normalization.
import datetime
from io import BytesIO
from typing import List, Optional, Union
from typing import Any
import openpyxl
from omniread.core.content import Content
from .parser_base import XlsxParserBase
class XlsxParser(XlsxParserBase):
class XlsxParser(XlsxParserBase[list[list[str]]]):
"""
Generic xlsx parser producing string rows from a worksheet.
@@ -41,7 +42,9 @@ class XlsxParser(XlsxParserBase):
locale-specific formatting must convert on their side.
"""
def __init__(self, content: Content, *, data_only: bool = True, read_only: bool = True):
def __init__(
self, content: Content, *, data_only: bool = True, read_only: bool = True
):
"""
Initialize the parser.
@@ -57,7 +60,7 @@ class XlsxParser(XlsxParserBase):
super().__init__(content)
self._data_only = data_only
self._read_only = read_only
self._workbook: Optional[openpyxl.Workbook] = None
self._workbook: openpyxl.Workbook | None = None
@property
def workbook(self) -> openpyxl.Workbook:
@@ -73,39 +76,39 @@ class XlsxParser(XlsxParserBase):
return self._workbook
@property
def sheet_names(self) -> List[str]:
def sheet_names(self) -> list[str]:
"""
Names of all worksheets contained in the workbook.
"""
return list(self.workbook.sheetnames)
def parse(self) -> List[List[str]]:
def parse(self) -> list[list[str]]:
"""
Parse the first worksheet into normalized string rows.
Returns:
List[List[str]]:
list[list[str]]:
Rows of the default (first) worksheet.
"""
return self.rows()
def rows(
self,
sheet: Optional[Union[int, str]] = None,
sheet: int | str | None = None,
*,
skip_empty: bool = True,
) -> List[List[str]]:
) -> list[list[str]]:
"""
Extract normalized string rows from a worksheet.
Args:
sheet (Optional[Union[int, str]]):
sheet (int | str | None):
Worksheet index or title; defaults to the first worksheet.
skip_empty (bool):
When True (default), rows whose cells are all blank are omitted.
Returns:
List[List[str]]:
list[list[str]]:
Normalized rows; trailing blank cells are trimmed per row.
Raises:
@@ -113,7 +116,7 @@ class XlsxParser(XlsxParserBase):
If the requested sheet does not exist.
"""
ws = self._resolve_sheet(sheet)
out: List[List[str]] = []
out: list[list[str]] = []
for row in ws.iter_rows(values_only=True):
cells = [self._cell_str(v) for v in row]
while cells and not cells[-1].strip():
@@ -123,7 +126,7 @@ class XlsxParser(XlsxParserBase):
out.append(cells)
return out
def _resolve_sheet(self, sheet: Optional[Union[int, str]]):
def _resolve_sheet(self, sheet: int | str | None) -> Any:
names = self.workbook.sheetnames
if not names:
raise ValueError("Workbook contains no worksheets")
@@ -138,7 +141,7 @@ class XlsxParser(XlsxParserBase):
return self.workbook[names[index]]
@staticmethod
def _cell_str(value) -> str:
def _cell_str(value: Any) -> str:
if value is None:
return ""
if isinstance(value, (datetime.datetime, datetime.date)):

18
omniread/xlsx/parser.pyi Normal file
View File

@@ -0,0 +1,18 @@
import openpyxl
from omniread.core.content import Content
from .parser_base import XlsxParserBase
class XlsxParser(XlsxParserBase[list[list[str]]]):
def __init__(
self, content: Content, *, data_only: bool = True, read_only: bool = True
) -> None: ...
@property
def workbook(self) -> openpyxl.Workbook: ...
@property
def sheet_names(self) -> list[str]: ...
def parse(self) -> list[list[str]]: ...
def rows(
self, sheet: int | str | None = None, *, skip_empty: bool = True
) -> list[list[str]]: ...

View File

@@ -8,8 +8,8 @@ format-agnostic `BaseParser` with constraints appropriate for Office Open
XML spreadsheet content.
"""
from typing import Generic, TypeVar
from abc import abstractmethod
from typing import Generic, TypeVar
from omniread.core.content import ContentType
from omniread.core.parser import BaseParser

View File

@@ -0,0 +1,13 @@
from abc import abstractmethod
from typing import Generic, TypeVar
from omniread.core.content import ContentType
from omniread.core.parser import BaseParser
T = TypeVar("T")
class XlsxParserBase(BaseParser[T], Generic[T]):
supported_types: set[ContentType]
@abstractmethod
def parse(self) -> T: ...

View File

@@ -10,9 +10,11 @@ The scraper does not interpret or parse the acquired bytes; it wraps them in
the canonical `Content` model.
"""
from typing import Any, Mapping, Optional
from collections.abc import Mapping
from typing import Any
from omniread.core.content import Content, ContentType
from .client import BaseXlsxClient
@@ -47,7 +49,7 @@ class XlsxScraper:
self,
source: Any,
*,
metadata: Optional[Mapping[str, Any]] = None,
metadata: Mapping[str, Any] | None = None,
) -> Content:
"""
Fetch an xlsx document from the given source.
@@ -56,7 +58,7 @@ class XlsxScraper:
source (Any):
Identifier of the spreadsheet source as understood by the
configured client.
metadata (Optional[Mapping[str, Any]], optional):
metadata (Mapping[str, Any] | None, optional):
Optional metadata to attach to the returned content.
Returns:

12
omniread/xlsx/scraper.pyi Normal file
View File

@@ -0,0 +1,12 @@
from collections.abc import Mapping
from typing import Any
from omniread.core.content import Content
from .client import BaseXlsxClient
class XlsxScraper:
def __init__(self, *, client: BaseXlsxClient) -> None: ...
def fetch(
self, source: Any, *, metadata: Mapping[str, Any] | None = ...
) -> Content: ...

Some files were not shown because too many files have changed in this diff Show More