From 650d3a19e84c92de185536e06c193e0c3231cf58 Mon Sep 17 00:00:00 2001 From: Vishesh 'ironeagle' Bangotra Date: Sat, 17 Oct 2026 02:16:48 +0530 Subject: [PATCH] chore(release): 0.0.3 Squash-merge lib-standardisation: adopt repo standards across packaging, tooling, docs, and CI. - standardize pyproject.toml (canonical packaging, lint config, extras, GSDFC pydoclint) - add py.typed marker, modernize type hints and Google-style docstrings - nest generated docs under docs/lib and docs/mcp; add hand-written wiki - adopt standard Type A PyPI CI pipeline (tag==version guard + quality gate) - promote generated CHANGELOG.md; bump version to 0.0.3 --- .drone.yml | 74 +- .gitignore | 10 +- CHANGELOG.md | 32 + LICENSE | 21 + README.md | 3 + docforge.nav.yml | 39 +- docs/{ => lib}/core/content.md | 0 docs/{ => lib}/core/index.md | 0 docs/{ => lib}/core/parser.md | 0 docs/{ => lib}/core/scraper.md | 0 docs/lib/csv/client.md | 3 + docs/lib/csv/index.md | 3 + docs/lib/csv/parser.md | 3 + docs/lib/csv/parser_base.md | 3 + docs/lib/csv/scraper.md | 3 + docs/{ => lib}/html/index.md | 0 docs/{ => lib}/html/parser.md | 0 docs/{ => lib}/html/scraper.md | 0 docs/{ => lib}/index.md | 0 docs/{ => lib}/pdf/client.md | 0 docs/{ => lib}/pdf/index.md | 0 docs/{ => lib}/pdf/parser.md | 0 docs/{ => lib}/pdf/scraper.md | 0 docs/lib/xlsx/client.md | 3 + docs/lib/xlsx/index.md | 3 + docs/lib/xlsx/parser.md | 3 + docs/lib/xlsx/parser_base.md | 3 + docs/lib/xlsx/scraper.md | 3 + {mcp_docs => docs/mcp}/index.json | 4 +- .../mcp}/modules/omniread.core.content.json | 53 +- .../mcp}/modules/omniread.core.json | 229 +- .../mcp}/modules/omniread.core.parser.json | 75 +- .../mcp}/modules/omniread.core.scraper.json | 51 +- docs/mcp/modules/omniread.csv.client.json | 41 + docs/mcp/modules/omniread.csv.json | 492 +++ docs/mcp/modules/omniread.csv.parser.json | 92 + .../mcp/modules/omniread.csv.parser_base.json | 127 + docs/mcp/modules/omniread.csv.scraper.json | 129 + .../mcp}/modules/omniread.html.json | 287 +- .../mcp}/modules/omniread.html.parser.json | 185 +- .../mcp}/modules/omniread.html.scraper.json | 76 +- docs/mcp/modules/omniread.json | 2699 +++++++++++++++++ .../mcp}/modules/omniread.pdf.client.json | 36 +- .../mcp}/modules/omniread.pdf.json | 182 +- .../mcp}/modules/omniread.pdf.parser.json | 59 +- .../mcp}/modules/omniread.pdf.scraper.json | 71 +- docs/mcp/modules/omniread.xlsx.client.json | 41 + docs/mcp/modules/omniread.xlsx.json | 520 ++++ docs/mcp/modules/omniread.xlsx.parser.json | 106 + .../modules/omniread.xlsx.parser_base.json | 127 + docs/mcp/modules/omniread.xlsx.scraper.json | 129 + docs/mcp/nav.json | 90 + mkdocs.yml => docs/mkdocs.lib.yml | 60 +- docs/mkdocs.wiki.yml | 69 + docs/wiki/01_overview.md | 86 + docs/wiki/02_how_to_use.md | 109 + docs/wiki/03_extending.md | 104 + docs/wiki/04_development.md | 87 + docs/wiki/index.md | 64 + mcp_docs/modules/omniread.json | 1639 ---------- mcp_docs/nav.json | 50 - omniread/__init__.py | 4 - omniread/__init__.pyi | 4 +- omniread/core/content.py | 9 +- omniread/core/content.pyi | 17 +- omniread/core/parser.py | 4 +- omniread/core/parser.pyi | 5 +- omniread/core/scraper.py | 7 +- omniread/core/scraper.pyi | 8 +- omniread/csv/__init__.pyi | 12 + omniread/csv/client.py | 2 +- omniread/csv/client.pyi | 10 + omniread/csv/parser.py | 16 +- omniread/csv/parser.pyi | 8 + omniread/csv/parser_base.py | 2 +- omniread/csv/parser_base.pyi | 13 + omniread/csv/scraper.py | 8 +- omniread/csv/scraper.pyi | 12 + omniread/html/__init__.py | 1 - omniread/html/__init__.pyi | 2 +- omniread/html/parser.py | 13 +- omniread/html/parser.pyi | 10 +- omniread/html/scraper.py | 16 +- omniread/html/scraper.pyi | 18 +- omniread/pdf/__init__.pyi | 2 +- omniread/pdf/client.py | 2 +- omniread/pdf/parser.py | 2 +- omniread/pdf/parser.pyi | 1 + omniread/pdf/scraper.py | 7 +- omniread/pdf/scraper.pyi | 11 +- omniread/py.typed | 0 omniread/xlsx/__init__.pyi | 12 + omniread/xlsx/client.py | 2 +- omniread/xlsx/client.pyi | 10 + omniread/xlsx/parser.py | 31 +- omniread/xlsx/parser.pyi | 18 + omniread/xlsx/parser_base.py | 2 +- omniread/xlsx/parser_base.pyi | 13 + omniread/xlsx/scraper.py | 8 +- omniread/xlsx/scraper.pyi | 12 + pyproject.toml | 87 +- tests/conftest.py | 16 +- tests/test_html_simple.py | 13 +- tests/test_html_table.py | 5 +- tests/test_pdf_simple.py | 3 +- tests/test_xlsx_simple.py | 55 +- 106 files changed, 6175 insertions(+), 2716 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 LICENSE rename docs/{ => lib}/core/content.md (100%) rename docs/{ => lib}/core/index.md (100%) rename docs/{ => lib}/core/parser.md (100%) rename docs/{ => lib}/core/scraper.md (100%) create mode 100644 docs/lib/csv/client.md create mode 100644 docs/lib/csv/index.md create mode 100644 docs/lib/csv/parser.md create mode 100644 docs/lib/csv/parser_base.md create mode 100644 docs/lib/csv/scraper.md rename docs/{ => lib}/html/index.md (100%) rename docs/{ => lib}/html/parser.md (100%) rename docs/{ => lib}/html/scraper.md (100%) rename docs/{ => lib}/index.md (100%) rename docs/{ => lib}/pdf/client.md (100%) rename docs/{ => lib}/pdf/index.md (100%) rename docs/{ => lib}/pdf/parser.md (100%) rename docs/{ => lib}/pdf/scraper.md (100%) create mode 100644 docs/lib/xlsx/client.md create mode 100644 docs/lib/xlsx/index.md create mode 100644 docs/lib/xlsx/parser.md create mode 100644 docs/lib/xlsx/parser_base.md create mode 100644 docs/lib/xlsx/scraper.md rename {mcp_docs => docs/mcp}/index.json (52%) rename {mcp_docs => docs/mcp}/modules/omniread.core.content.json (72%) rename {mcp_docs => docs/mcp}/modules/omniread.core.json (68%) rename {mcp_docs => docs/mcp}/modules/omniread.core.parser.json (69%) rename {mcp_docs => docs/mcp}/modules/omniread.core.scraper.json (58%) create mode 100644 docs/mcp/modules/omniread.csv.client.json create mode 100644 docs/mcp/modules/omniread.csv.json create mode 100644 docs/mcp/modules/omniread.csv.parser.json create mode 100644 docs/mcp/modules/omniread.csv.parser_base.json create mode 100644 docs/mcp/modules/omniread.csv.scraper.json rename {mcp_docs => docs/mcp}/modules/omniread.html.json (67%) rename {mcp_docs => docs/mcp}/modules/omniread.html.parser.json (66%) rename {mcp_docs => docs/mcp}/modules/omniread.html.scraper.json (64%) create mode 100644 docs/mcp/modules/omniread.json rename {mcp_docs => docs/mcp}/modules/omniread.pdf.client.json (66%) rename {mcp_docs => docs/mcp}/modules/omniread.pdf.json (69%) rename {mcp_docs => docs/mcp}/modules/omniread.pdf.parser.json (72%) rename {mcp_docs => docs/mcp}/modules/omniread.pdf.scraper.json (65%) create mode 100644 docs/mcp/modules/omniread.xlsx.client.json create mode 100644 docs/mcp/modules/omniread.xlsx.json create mode 100644 docs/mcp/modules/omniread.xlsx.parser.json create mode 100644 docs/mcp/modules/omniread.xlsx.parser_base.json create mode 100644 docs/mcp/modules/omniread.xlsx.scraper.json create mode 100644 docs/mcp/nav.json rename mkdocs.yml => docs/mkdocs.lib.yml (79%) create mode 100644 docs/mkdocs.wiki.yml create mode 100644 docs/wiki/01_overview.md create mode 100644 docs/wiki/02_how_to_use.md create mode 100644 docs/wiki/03_extending.md create mode 100644 docs/wiki/04_development.md create mode 100644 docs/wiki/index.md delete mode 100644 mcp_docs/modules/omniread.json delete mode 100644 mcp_docs/nav.json create mode 100644 omniread/csv/__init__.pyi create mode 100644 omniread/csv/client.pyi create mode 100644 omniread/csv/parser.pyi create mode 100644 omniread/csv/parser_base.pyi create mode 100644 omniread/csv/scraper.pyi create mode 100644 omniread/py.typed create mode 100644 omniread/xlsx/__init__.pyi create mode 100644 omniread/xlsx/client.pyi create mode 100644 omniread/xlsx/parser.pyi create mode 100644 omniread/xlsx/parser_base.pyi create mode 100644 omniread/xlsx/scraper.pyi diff --git a/.drone.yml b/.drone.yml index ddd0ac4..122d0f6 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,3 +1,21 @@ +# Drone standard — Type A: PyPI package +# +# Canonical pipeline for internal Python libraries that ship a wheel/sdist to +# the private index (dagpipe, doc-forge, hexa, mail-intake, mongo-ops, +# omniread, openapi-first, py-jwt). +# +# Two pipelines ship together: +# build-and-publish-pypi — tag-triggered build + quality gate + publish. +# backfill-pypi-from-tags — custom-triggered replay of every git tag that is +# missing from the index (disaster recovery). +# +# Secrets (Drone): PIP_REPO_URL, PIP_USERNAME, PIP_PASSWORD. +# Copy to /.drone.yml unchanged; the package name and version are read +# from pyproject.toml, so no per-repo edits are required. +# +# Version guard: the tag build fails unless the git tag equals the version in +# pyproject.toml (see the versioning standard). This stops a tag from looking +# like a release while silently publishing nothing. --- kind: pipeline type: docker @@ -23,20 +41,46 @@ steps: commands: - PACKAGE_NAME=$(grep -E '^name\s*=' pyproject.toml | head -1 | cut -d'"' -f2) - VERSION=$(grep -E '^version\s*=' pyproject.toml | head -1 | cut -d'"' -f2) - - echo "🔍 Checking if $PACKAGE_NAME==$VERSION exists on $PIP_REPO_URL ..." - | - if curl -fsSL -u "$PIP_USERNAME:$PIP_PASSWORD" "$PIP_REPO_URL/simple/$PACKAGE_NAME/" | grep -q "$VERSION"; then - echo "✅ $PACKAGE_NAME==$VERSION already exists — skipping build." + if [ "$DRONE_TAG" != "$VERSION" ]; then + echo "Tag '$DRONE_TAG' does not match pyproject version '$VERSION'; refusing to publish." + exit 1 + fi + - echo "Checking if $PACKAGE_NAME==$VERSION exists on $PIP_REPO_URL ..." + - | + if curl -fsSL -u "$PIP_USERNAME:$PIP_PASSWORD" "$PIP_REPO_URL/simple/$PACKAGE_NAME/" | grep -qE "${PACKAGE_NAME}-${VERSION}(-|\.tar\.gz|\.whl)"; then + echo "$PACKAGE_NAME==$VERSION already exists; skipping build." exit 78 else - echo "🆕 New version detected: $PACKAGE_NAME==$VERSION" + echo "New version detected: $PACKAGE_NAME==$VERSION" fi + - name: quality-gate + image: python:3.13-slim + environment: + PIP_REPO_URL: + from_secret: PIP_REPO_URL + PIP_USERNAME: + from_secret: PIP_USERNAME + PIP_PASSWORD: + from_secret: PIP_PASSWORD + commands: + - pip install --upgrade pip build + - | + AUTH_URL="https://${PIP_USERNAME}:${PIP_PASSWORD}@$(echo "${PIP_REPO_URL#*://}" | sed 's:/*$::')/simple" + pip install --index-url "$AUTH_URL" --extra-index-url https://pypi.org/simple/ -U ".[dev]" "pydoclint>=0.9.1" + - echo "Running quality gate..." + - python -m black --check . + - python -m ruff check . + - python -m mypy + - pydoclint . + - python -m pytest + - name: build-package image: python:3.13-slim commands: - pip install --upgrade pip build - - echo "📦 Building Python package..." + - echo "Building Python package..." - python -m build - ls -l dist @@ -51,7 +95,7 @@ steps: from_secret: PIP_PASSWORD commands: - pip install --upgrade twine - - echo "🚀 Uploading to private PyPI at $PIP_REPO_URL ..." + - echo "Uploading to private PyPI at $PIP_REPO_URL ..." - | twine upload \ --repository-url "$PIP_REPO_URL" \ @@ -98,25 +142,31 @@ steps: set -e PACKAGE_NAME=$(grep -E '^name\s*=' pyproject.toml | cut -d'"' -f2) - echo "📦 Package: $PACKAGE_NAME" + echo "Package: $PACKAGE_NAME" for TAG in $(git tag --sort=version:refname); do VERSION="$TAG" - echo "🔁 Version: $VERSION" + echo "Version: $VERSION" if curl -fsSL -u "$PIP_USERNAME:$PIP_PASSWORD" \ - "$PIP_REPO_URL/simple/$PACKAGE_NAME/" | grep -q "$VERSION"; then - echo "⏭️ Exists, skipping" + "$PIP_REPO_URL/simple/$PACKAGE_NAME/" | grep -qE "${PACKAGE_NAME}-${VERSION}(-|\.tar\.gz|\.whl)"; then + echo "Exists, skipping" continue fi git checkout --force "$TAG" - echo "🏗️ Building $VERSION" + TAG_VERSION=$(grep -E '^version\s*=' pyproject.toml | head -1 | cut -d'"' -f2) + if [ "$TAG" != "$TAG_VERSION" ]; then + echo "Tag $TAG != pyproject version $TAG_VERSION at that commit; skipping" + continue + fi + + echo "Building $VERSION" rm -rf dist python -m build - echo "⬆️ Uploading $VERSION" + echo "Uploading $VERSION" twine upload \ --repository-url "$PIP_REPO_URL" \ -u "$PIP_USERNAME" \ diff --git a/.gitignore b/.gitignore index 13b61a5..600ec58 100644 --- a/.gitignore +++ b/.gitignore @@ -25,10 +25,17 @@ env/ htmlcov/ coverage.xml +# Lint / type checking +.mypy_cache/ +.ruff_cache/ + # Environment variables .env .env.local +# Docs build artifacts +site + # OS .DS_Store Thumbs.db @@ -37,5 +44,4 @@ Thumbs.db *.swp *.swo *~ -*.tmp -site +*.tmp \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..b1b0bde --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,32 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [0.0.3] - 2026-10-17 + +### Added +- `py.typed` marker for PEP 561 type information. +- Type stubs for the `csv` and `xlsx` subpackages. +- Hand-written wiki, plus doc-forge-generated API reference under `docs/lib` + and MCP bundle under `docs/mcp` (per-kind MkDocs configs). +- MIT `LICENSE`. + +### Changed +- Adopted the standard Type A PyPI CI pipeline: a `tag == version` guard and a + quality gate (black, ruff, mypy, pydoclint, pytest). +- Standardized `pyproject.toml` (canonical packaging, lint tool config, extras) + with a shared Google-style `[tool.pydoclint]` configuration. +- Docstring types modernised (`List` -> `list`, `Optional` -> `X | None`) to + match signatures. + +### Fixed +- `XLSX`/`CSV` members added to the `content` type surface. +- Stub fixes for typed API surfaces. + +[Unreleased]: https://git.aetoskia.com/aetos/omniread/compare/0.0.3...HEAD +[0.0.3]: https://git.aetoskia.com/aetos/omniread/compare/0.0.2...0.0.3 diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..87328b2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Aetoskia Platform + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index 0a66f72..530daa1 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,9 @@ required. - `FileSystemPDFClient`: Local filesystem PDF access. - `PDFScraper`: PDF-specific content acquisition. - `PDFParser`: Base parser for PDF binary interpretation. +- `FileSystemXlsxClient`: Local filesystem spreadsheet access. +- `XlsxScraper`: XLSX-specific content acquisition. +- `XlsxParser`: Generic string-row parser for xlsx workbooks. --- diff --git a/docforge.nav.yml b/docforge.nav.yml index 20349e8..2532b93 100644 --- a/docforge.nav.yml +++ b/docforge.nav.yml @@ -1,16 +1,31 @@ -home: index.md +home: lib/index.md groups: Core API: - - core/index.md - - core/content.md - - core/parser.md - - core/scraper.md + - lib/core/index.md + - lib/core/content.md + - lib/core/parser.md + - lib/core/scraper.md HTML Handling: - - html/index.md - - html/parser.md - - html/scraper.md + - lib/html/index.md + - lib/html/parser.md + - lib/html/scraper.md PDF Handling: - - pdf/index.md - - pdf/client.md - - pdf/parser.md - - pdf/scraper.md + - lib/pdf/index.md + - lib/pdf/client.md + - lib/pdf/parser.md + - lib/pdf/scraper.md + CSV Handling: + - lib/csv/index.md + - lib/csv/client.md + - lib/csv/parser_base.md + - lib/csv/parser.md + - lib/csv/scraper.md + XLSX Handling: + - lib/xlsx/index.md + - lib/xlsx/client.md + - lib/xlsx/parser_base.md + - lib/xlsx/parser.md + - lib/xlsx/scraper.md +icon: + logo: material/book-open-page-variant + repo: fontawesome/brands/github \ No newline at end of file diff --git a/docs/core/content.md b/docs/lib/core/content.md similarity index 100% rename from docs/core/content.md rename to docs/lib/core/content.md diff --git a/docs/core/index.md b/docs/lib/core/index.md similarity index 100% rename from docs/core/index.md rename to docs/lib/core/index.md diff --git a/docs/core/parser.md b/docs/lib/core/parser.md similarity index 100% rename from docs/core/parser.md rename to docs/lib/core/parser.md diff --git a/docs/core/scraper.md b/docs/lib/core/scraper.md similarity index 100% rename from docs/core/scraper.md rename to docs/lib/core/scraper.md diff --git a/docs/lib/csv/client.md b/docs/lib/csv/client.md new file mode 100644 index 0000000..192c801 --- /dev/null +++ b/docs/lib/csv/client.md @@ -0,0 +1,3 @@ +# Client + +::: omniread.csv.client diff --git a/docs/lib/csv/index.md b/docs/lib/csv/index.md new file mode 100644 index 0000000..5459b10 --- /dev/null +++ b/docs/lib/csv/index.md @@ -0,0 +1,3 @@ +# Csv + +::: omniread.csv diff --git a/docs/lib/csv/parser.md b/docs/lib/csv/parser.md new file mode 100644 index 0000000..cb84ff6 --- /dev/null +++ b/docs/lib/csv/parser.md @@ -0,0 +1,3 @@ +# Parser + +::: omniread.csv.parser diff --git a/docs/lib/csv/parser_base.md b/docs/lib/csv/parser_base.md new file mode 100644 index 0000000..db71308 --- /dev/null +++ b/docs/lib/csv/parser_base.md @@ -0,0 +1,3 @@ +# Parser Base + +::: omniread.csv.parser_base diff --git a/docs/lib/csv/scraper.md b/docs/lib/csv/scraper.md new file mode 100644 index 0000000..a6cd836 --- /dev/null +++ b/docs/lib/csv/scraper.md @@ -0,0 +1,3 @@ +# Scraper + +::: omniread.csv.scraper diff --git a/docs/html/index.md b/docs/lib/html/index.md similarity index 100% rename from docs/html/index.md rename to docs/lib/html/index.md diff --git a/docs/html/parser.md b/docs/lib/html/parser.md similarity index 100% rename from docs/html/parser.md rename to docs/lib/html/parser.md diff --git a/docs/html/scraper.md b/docs/lib/html/scraper.md similarity index 100% rename from docs/html/scraper.md rename to docs/lib/html/scraper.md diff --git a/docs/index.md b/docs/lib/index.md similarity index 100% rename from docs/index.md rename to docs/lib/index.md diff --git a/docs/pdf/client.md b/docs/lib/pdf/client.md similarity index 100% rename from docs/pdf/client.md rename to docs/lib/pdf/client.md diff --git a/docs/pdf/index.md b/docs/lib/pdf/index.md similarity index 100% rename from docs/pdf/index.md rename to docs/lib/pdf/index.md diff --git a/docs/pdf/parser.md b/docs/lib/pdf/parser.md similarity index 100% rename from docs/pdf/parser.md rename to docs/lib/pdf/parser.md diff --git a/docs/pdf/scraper.md b/docs/lib/pdf/scraper.md similarity index 100% rename from docs/pdf/scraper.md rename to docs/lib/pdf/scraper.md diff --git a/docs/lib/xlsx/client.md b/docs/lib/xlsx/client.md new file mode 100644 index 0000000..43ad64e --- /dev/null +++ b/docs/lib/xlsx/client.md @@ -0,0 +1,3 @@ +# Client + +::: omniread.xlsx.client diff --git a/docs/lib/xlsx/index.md b/docs/lib/xlsx/index.md new file mode 100644 index 0000000..017f313 --- /dev/null +++ b/docs/lib/xlsx/index.md @@ -0,0 +1,3 @@ +# Xlsx + +::: omniread.xlsx diff --git a/docs/lib/xlsx/parser.md b/docs/lib/xlsx/parser.md new file mode 100644 index 0000000..7eb915a --- /dev/null +++ b/docs/lib/xlsx/parser.md @@ -0,0 +1,3 @@ +# Parser + +::: omniread.xlsx.parser diff --git a/docs/lib/xlsx/parser_base.md b/docs/lib/xlsx/parser_base.md new file mode 100644 index 0000000..5cf91ea --- /dev/null +++ b/docs/lib/xlsx/parser_base.md @@ -0,0 +1,3 @@ +# Parser Base + +::: omniread.xlsx.parser_base diff --git a/docs/lib/xlsx/scraper.md b/docs/lib/xlsx/scraper.md new file mode 100644 index 0000000..c374fb7 --- /dev/null +++ b/docs/lib/xlsx/scraper.md @@ -0,0 +1,3 @@ +# Scraper + +::: omniread.xlsx.scraper diff --git a/mcp_docs/index.json b/docs/mcp/index.json similarity index 52% rename from mcp_docs/index.json rename to docs/mcp/index.json index 48b0a19..c73df60 100644 --- a/mcp_docs/index.json +++ b/docs/mcp/index.json @@ -1,6 +1,6 @@ { - "project": "omniread", + "project": "OmniRead", "type": "docforge-model", - "modules_count": 12, + "modules_count": 22, "source": "docforge" } \ No newline at end of file diff --git a/mcp_docs/modules/omniread.core.content.json b/docs/mcp/modules/omniread.core.content.json similarity index 72% rename from mcp_docs/modules/omniread.core.content.json rename to docs/mcp/modules/omniread.core.content.json index 54a56c3..a71ec4f 100644 --- a/mcp_docs/modules/omniread.core.content.json +++ b/docs/mcp/modules/omniread.core.content.json @@ -4,46 +4,11 @@ "path": "omniread.core.content", "docstring": "# Summary\n\nCanonical content models for OmniRead.\n\nThis module defines the **format-agnostic content representation** used across\nall parsers and scrapers in OmniRead.\n\nThe models defined here represent *what* was extracted, not *how* it was\nretrieved or parsed. Format-specific behavior and metadata must not alter\nthe semantic meaning of these models.", "objects": { - "Enum": { - "name": "Enum", - "kind": "alias", - "path": "omniread.core.content.Enum", - "signature": "", - "docstring": null - }, - "dataclass": { - "name": "dataclass", - "kind": "alias", - "path": "omniread.core.content.dataclass", - "signature": "", - "docstring": null - }, - "Any": { - "name": "Any", - "kind": "alias", - "path": "omniread.core.content.Any", - "signature": "", - "docstring": null - }, - "Mapping": { - "name": "Mapping", - "kind": "alias", - "path": "omniread.core.content.Mapping", - "signature": "", - "docstring": null - }, - "Optional": { - "name": "Optional", - "kind": "alias", - "path": "omniread.core.content.Optional", - "signature": "", - "docstring": null - }, "ContentType": { "name": "ContentType", "kind": "class", "path": "omniread.core.content.ContentType", - "signature": "", + "signature": null, "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { @@ -60,6 +25,20 @@ "signature": null, "docstring": "PDF document content." }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.core.content.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.core.content.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, "JSON": { "name": "JSON", "kind": "attribute", @@ -80,7 +59,7 @@ "name": "Content", "kind": "class", "path": "omniread.core.content.Content", - "signature": "", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { diff --git a/mcp_docs/modules/omniread.core.json b/docs/mcp/modules/omniread.core.json similarity index 68% rename from mcp_docs/modules/omniread.core.json rename to docs/mcp/modules/omniread.core.json index 2cd7094..fd51d72 100644 --- a/mcp_docs/modules/omniread.core.json +++ b/docs/mcp/modules/omniread.core.json @@ -8,35 +8,35 @@ "name": "Content", "kind": "class", "path": "omniread.core.Content", - "signature": "", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.core.Content.raw", - "signature": "", + "signature": null, "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.core.Content.source", - "signature": "", + "signature": null, "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.core.Content.content_type", - "signature": "", + "signature": null, "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.core.Content.metadata", - "signature": "", + "signature": null, "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } @@ -45,35 +45,49 @@ "name": "ContentType", "kind": "class", "path": "omniread.core.ContentType", - "signature": "", + "signature": null, "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", "kind": "attribute", "path": "omniread.core.ContentType.HTML", - "signature": "", + "signature": null, "docstring": "HTML document content." }, "PDF": { "name": "PDF", "kind": "attribute", "path": "omniread.core.ContentType.PDF", - "signature": "", + "signature": null, "docstring": "PDF document content." }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.core.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.core.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, "JSON": { "name": "JSON", "kind": "attribute", "path": "omniread.core.ContentType.JSON", - "signature": "", + "signature": null, "docstring": "JSON document content." }, "XML": { "name": "XML", "kind": "attribute", "path": "omniread.core.ContentType.XML", - "signature": "", + "signature": null, "docstring": "XML document content." } } @@ -82,35 +96,35 @@ "name": "BaseParser", "kind": "class", "path": "omniread.core.BaseParser", - "signature": "", + "signature": "BaseParser(content: Content)", "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", "members": { "supported_types": { "name": "supported_types", "kind": "attribute", "path": "omniread.core.BaseParser.supported_types", - "signature": "", + "signature": null, "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." }, "content": { "name": "content", "kind": "attribute", "path": "omniread.core.BaseParser.content", - "signature": "", + "signature": null, "docstring": null }, "parse": { "name": "parse", "kind": "function", "path": "omniread.core.BaseParser.parse", - "signature": "", + "signature": "parse()", "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." }, "supports": { "name": "supports", "kind": "function", "path": "omniread.core.BaseParser.supports", - "signature": "", + "signature": "supports()", "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." } } @@ -119,15 +133,15 @@ "name": "BaseScraper", "kind": "class", "path": "omniread.core.BaseScraper", - "signature": "", + "signature": null, "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.core.BaseScraper.fetch", - "signature": "", - "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." + "signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None)", + "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." } } }, @@ -138,46 +152,11 @@ "signature": null, "docstring": "# Summary\n\nCanonical content models for OmniRead.\n\nThis module defines the **format-agnostic content representation** used across\nall parsers and scrapers in OmniRead.\n\nThe models defined here represent *what* was extracted, not *how* it was\nretrieved or parsed. Format-specific behavior and metadata must not alter\nthe semantic meaning of these models.", "members": { - "Enum": { - "name": "Enum", - "kind": "alias", - "path": "omniread.core.content.Enum", - "signature": "", - "docstring": null - }, - "dataclass": { - "name": "dataclass", - "kind": "alias", - "path": "omniread.core.content.dataclass", - "signature": "", - "docstring": null - }, - "Any": { - "name": "Any", - "kind": "alias", - "path": "omniread.core.content.Any", - "signature": "", - "docstring": null - }, - "Mapping": { - "name": "Mapping", - "kind": "alias", - "path": "omniread.core.content.Mapping", - "signature": "", - "docstring": null - }, - "Optional": { - "name": "Optional", - "kind": "alias", - "path": "omniread.core.content.Optional", - "signature": "", - "docstring": null - }, "ContentType": { "name": "ContentType", "kind": "class", "path": "omniread.core.content.ContentType", - "signature": "", + "signature": null, "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { @@ -194,6 +173,20 @@ "signature": null, "docstring": "PDF document content." }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.core.content.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.core.content.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, "JSON": { "name": "JSON", "kind": "attribute", @@ -214,7 +207,7 @@ "name": "Content", "kind": "class", "path": "omniread.core.content.Content", - "signature": "", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { @@ -256,74 +249,39 @@ "signature": null, "docstring": "# Summary\n\nAbstract parsing contracts for OmniRead.\n\nThis module defines the **format-agnostic parser interface** used to transform\nraw content into structured, typed representations.\n\nParsers are responsible for:\n\n- Interpreting a single `Content` instance\n- Validating compatibility with the content type\n- Producing a structured output suitable for downstream consumers\n\nParsers are not responsible for:\n\n- Fetching or acquiring content\n- Performing retries or error recovery\n- Managing multiple content sources", "members": { - "ABC": { - "name": "ABC", - "kind": "alias", - "path": "omniread.core.parser.ABC", - "signature": "", - "docstring": null - }, - "abstractmethod": { - "name": "abstractmethod", - "kind": "alias", - "path": "omniread.core.parser.abstractmethod", - "signature": "", - "docstring": null - }, - "Generic": { - "name": "Generic", - "kind": "alias", - "path": "omniread.core.parser.Generic", - "signature": "", - "docstring": null - }, - "TypeVar": { - "name": "TypeVar", - "kind": "alias", - "path": "omniread.core.parser.TypeVar", - "signature": "", - "docstring": null - }, - "Set": { - "name": "Set", - "kind": "alias", - "path": "omniread.core.parser.Set", - "signature": "", - "docstring": null - }, "Content": { "name": "Content", "kind": "class", "path": "omniread.core.parser.Content", - "signature": "", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.core.parser.Content.raw", - "signature": "", + "signature": null, "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.core.parser.Content.source", - "signature": "", + "signature": null, "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.core.parser.Content.content_type", - "signature": "", + "signature": null, "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.core.parser.Content.metadata", - "signature": "", + "signature": null, "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } @@ -332,35 +290,49 @@ "name": "ContentType", "kind": "class", "path": "omniread.core.parser.ContentType", - "signature": "", + "signature": null, "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", "kind": "attribute", "path": "omniread.core.parser.ContentType.HTML", - "signature": "", + "signature": null, "docstring": "HTML document content." }, "PDF": { "name": "PDF", "kind": "attribute", "path": "omniread.core.parser.ContentType.PDF", - "signature": "", + "signature": null, "docstring": "PDF document content." }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.core.parser.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.core.parser.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, "JSON": { "name": "JSON", "kind": "attribute", "path": "omniread.core.parser.ContentType.JSON", - "signature": "", + "signature": null, "docstring": "JSON document content." }, "XML": { "name": "XML", "kind": "attribute", "path": "omniread.core.parser.ContentType.XML", - "signature": "", + "signature": null, "docstring": "XML document content." } } @@ -376,7 +348,7 @@ "name": "BaseParser", "kind": "class", "path": "omniread.core.parser.BaseParser", - "signature": "", + "signature": "BaseParser(content: Content)", "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", "members": { "supported_types": { @@ -397,14 +369,14 @@ "name": "parse", "kind": "function", "path": "omniread.core.parser.BaseParser.parse", - "signature": "", + "signature": "parse() -> T", "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." }, "supports": { "name": "supports", "kind": "function", "path": "omniread.core.parser.BaseParser.supports", - "signature": "", + "signature": "supports() -> bool", "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." } } @@ -418,74 +390,39 @@ "signature": null, "docstring": "# Summary\n\nAbstract scraping contracts for OmniRead.\n\nThis module defines the **format-agnostic scraper interface** responsible for\nacquiring raw content from external sources.\n\nScrapers are responsible for:\n\n- Locating and retrieving raw content bytes\n- Attaching minimal contextual metadata\n- Returning normalized `Content` objects\n\nScrapers are explicitly NOT responsible for:\n\n- Parsing or interpreting content\n- Inferring structure or semantics\n- Performing content-type specific processing\n\nAll interpretation must be delegated to parsers.", "members": { - "ABC": { - "name": "ABC", - "kind": "alias", - "path": "omniread.core.scraper.ABC", - "signature": "", - "docstring": null - }, - "abstractmethod": { - "name": "abstractmethod", - "kind": "alias", - "path": "omniread.core.scraper.abstractmethod", - "signature": "", - "docstring": null - }, - "Any": { - "name": "Any", - "kind": "alias", - "path": "omniread.core.scraper.Any", - "signature": "", - "docstring": null - }, - "Mapping": { - "name": "Mapping", - "kind": "alias", - "path": "omniread.core.scraper.Mapping", - "signature": "", - "docstring": null - }, - "Optional": { - "name": "Optional", - "kind": "alias", - "path": "omniread.core.scraper.Optional", - "signature": "", - "docstring": null - }, "Content": { "name": "Content", "kind": "class", "path": "omniread.core.scraper.Content", - "signature": "", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.core.scraper.Content.raw", - "signature": "", + "signature": null, "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.core.scraper.Content.source", - "signature": "", + "signature": null, "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.core.scraper.Content.content_type", - "signature": "", + "signature": null, "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.core.scraper.Content.metadata", - "signature": "", + "signature": null, "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } @@ -494,15 +431,15 @@ "name": "BaseScraper", "kind": "class", "path": "omniread.core.scraper.BaseScraper", - "signature": "", + "signature": null, "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.core.scraper.BaseScraper.fetch", - "signature": "", - "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." + "signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None) -> Content", + "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." } } } diff --git a/mcp_docs/modules/omniread.core.parser.json b/docs/mcp/modules/omniread.core.parser.json similarity index 69% rename from mcp_docs/modules/omniread.core.parser.json rename to docs/mcp/modules/omniread.core.parser.json index 6df349d..88c67ce 100644 --- a/mcp_docs/modules/omniread.core.parser.json +++ b/docs/mcp/modules/omniread.core.parser.json @@ -4,74 +4,39 @@ "path": "omniread.core.parser", "docstring": "# Summary\n\nAbstract parsing contracts for OmniRead.\n\nThis module defines the **format-agnostic parser interface** used to transform\nraw content into structured, typed representations.\n\nParsers are responsible for:\n\n- Interpreting a single `Content` instance\n- Validating compatibility with the content type\n- Producing a structured output suitable for downstream consumers\n\nParsers are not responsible for:\n\n- Fetching or acquiring content\n- Performing retries or error recovery\n- Managing multiple content sources", "objects": { - "ABC": { - "name": "ABC", - "kind": "alias", - "path": "omniread.core.parser.ABC", - "signature": "", - "docstring": null - }, - "abstractmethod": { - "name": "abstractmethod", - "kind": "alias", - "path": "omniread.core.parser.abstractmethod", - "signature": "", - "docstring": null - }, - "Generic": { - "name": "Generic", - "kind": "alias", - "path": "omniread.core.parser.Generic", - "signature": "", - "docstring": null - }, - "TypeVar": { - "name": "TypeVar", - "kind": "alias", - "path": "omniread.core.parser.TypeVar", - "signature": "", - "docstring": null - }, - "Set": { - "name": "Set", - "kind": "alias", - "path": "omniread.core.parser.Set", - "signature": "", - "docstring": null - }, "Content": { "name": "Content", "kind": "class", "path": "omniread.core.parser.Content", - "signature": "", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.core.parser.Content.raw", - "signature": "", + "signature": null, "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.core.parser.Content.source", - "signature": "", + "signature": null, "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.core.parser.Content.content_type", - "signature": "", + "signature": null, "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.core.parser.Content.metadata", - "signature": "", + "signature": null, "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } @@ -80,35 +45,49 @@ "name": "ContentType", "kind": "class", "path": "omniread.core.parser.ContentType", - "signature": "", + "signature": null, "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", "kind": "attribute", "path": "omniread.core.parser.ContentType.HTML", - "signature": "", + "signature": null, "docstring": "HTML document content." }, "PDF": { "name": "PDF", "kind": "attribute", "path": "omniread.core.parser.ContentType.PDF", - "signature": "", + "signature": null, "docstring": "PDF document content." }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.core.parser.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.core.parser.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, "JSON": { "name": "JSON", "kind": "attribute", "path": "omniread.core.parser.ContentType.JSON", - "signature": "", + "signature": null, "docstring": "JSON document content." }, "XML": { "name": "XML", "kind": "attribute", "path": "omniread.core.parser.ContentType.XML", - "signature": "", + "signature": null, "docstring": "XML document content." } } @@ -124,7 +103,7 @@ "name": "BaseParser", "kind": "class", "path": "omniread.core.parser.BaseParser", - "signature": "", + "signature": "BaseParser(content: Content)", "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", "members": { "supported_types": { @@ -145,14 +124,14 @@ "name": "parse", "kind": "function", "path": "omniread.core.parser.BaseParser.parse", - "signature": "", + "signature": "parse() -> T", "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." }, "supports": { "name": "supports", "kind": "function", "path": "omniread.core.parser.BaseParser.supports", - "signature": "", + "signature": "supports() -> bool", "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." } } diff --git a/mcp_docs/modules/omniread.core.scraper.json b/docs/mcp/modules/omniread.core.scraper.json similarity index 58% rename from mcp_docs/modules/omniread.core.scraper.json rename to docs/mcp/modules/omniread.core.scraper.json index 2adcd11..5bb6bde 100644 --- a/mcp_docs/modules/omniread.core.scraper.json +++ b/docs/mcp/modules/omniread.core.scraper.json @@ -4,74 +4,39 @@ "path": "omniread.core.scraper", "docstring": "# Summary\n\nAbstract scraping contracts for OmniRead.\n\nThis module defines the **format-agnostic scraper interface** responsible for\nacquiring raw content from external sources.\n\nScrapers are responsible for:\n\n- Locating and retrieving raw content bytes\n- Attaching minimal contextual metadata\n- Returning normalized `Content` objects\n\nScrapers are explicitly NOT responsible for:\n\n- Parsing or interpreting content\n- Inferring structure or semantics\n- Performing content-type specific processing\n\nAll interpretation must be delegated to parsers.", "objects": { - "ABC": { - "name": "ABC", - "kind": "alias", - "path": "omniread.core.scraper.ABC", - "signature": "", - "docstring": null - }, - "abstractmethod": { - "name": "abstractmethod", - "kind": "alias", - "path": "omniread.core.scraper.abstractmethod", - "signature": "", - "docstring": null - }, - "Any": { - "name": "Any", - "kind": "alias", - "path": "omniread.core.scraper.Any", - "signature": "", - "docstring": null - }, - "Mapping": { - "name": "Mapping", - "kind": "alias", - "path": "omniread.core.scraper.Mapping", - "signature": "", - "docstring": null - }, - "Optional": { - "name": "Optional", - "kind": "alias", - "path": "omniread.core.scraper.Optional", - "signature": "", - "docstring": null - }, "Content": { "name": "Content", "kind": "class", "path": "omniread.core.scraper.Content", - "signature": "", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.core.scraper.Content.raw", - "signature": "", + "signature": null, "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.core.scraper.Content.source", - "signature": "", + "signature": null, "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.core.scraper.Content.content_type", - "signature": "", + "signature": null, "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.core.scraper.Content.metadata", - "signature": "", + "signature": null, "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } @@ -80,15 +45,15 @@ "name": "BaseScraper", "kind": "class", "path": "omniread.core.scraper.BaseScraper", - "signature": "", + "signature": null, "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.core.scraper.BaseScraper.fetch", - "signature": "", - "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." + "signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None) -> Content", + "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." } } } diff --git a/docs/mcp/modules/omniread.csv.client.json b/docs/mcp/modules/omniread.csv.client.json new file mode 100644 index 0000000..7d36d79 --- /dev/null +++ b/docs/mcp/modules/omniread.csv.client.json @@ -0,0 +1,41 @@ +{ + "module": "omniread.csv.client", + "content": { + "path": "omniread.csv.client", + "docstring": "# Summary\n\nCSV client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw\ncomma-separated-value document bytes from a concrete backing store.\n\nClients provide low-level access to csv binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems", + "objects": { + "BaseCsvClient": { + "name": "BaseCsvClient", + "kind": "class", + "path": "omniread.csv.client.BaseCsvClient", + "signature": null, + "docstring": "Abstract client responsible for retrieving csv bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full csv binary payload.\n - Raise retrieval-specific errors on failure.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.csv.client.BaseCsvClient.fetch", + "signature": "fetch(source: Any) -> bytes", + "docstring": "Fetch raw csv bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." + } + } + }, + "FileSystemCsvClient": { + "name": "FileSystemCsvClient", + "kind": "class", + "path": "omniread.csv.client.FileSystemCsvClient", + "signature": null, + "docstring": "CSV client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads csv files directly from the disk and\n returns their raw binary contents.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.csv.client.FileSystemCsvClient.fetch", + "signature": "fetch(path: Path) -> bytes", + "docstring": "Read a csv file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the csv file.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." + } + } + } + } + } +} \ No newline at end of file diff --git a/docs/mcp/modules/omniread.csv.json b/docs/mcp/modules/omniread.csv.json new file mode 100644 index 0000000..0dfa9aa --- /dev/null +++ b/docs/mcp/modules/omniread.csv.json @@ -0,0 +1,492 @@ +{ + "module": "omniread.csv", + "content": { + "path": "omniread.csv", + "docstring": "# Summary\n\nCSV subpackage for OmniRead.\n\nProvides acquisition and parsing of comma-separated-value content:\n\n- `BaseCsvClient`: abstract backing-store client for csv bytes.\n- `FileSystemCsvClient`: local filesystem implementation.\n- `CsvScraper`: wraps fetched bytes into canonical `Content`.\n- `CsvParserBase`: content-type-enforcing parser contract.\n- `CsvParser`: generic string-row parser built on the standard csv module.", + "objects": { + "BaseCsvClient": { + "name": "BaseCsvClient", + "kind": "class", + "path": "omniread.csv.BaseCsvClient", + "signature": null, + "docstring": "Abstract client responsible for retrieving csv bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full csv binary payload.\n - Raise retrieval-specific errors on failure.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.csv.BaseCsvClient.fetch", + "signature": "fetch(source: Any)", + "docstring": "Fetch raw csv bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." + } + } + }, + "FileSystemCsvClient": { + "name": "FileSystemCsvClient", + "kind": "class", + "path": "omniread.csv.FileSystemCsvClient", + "signature": null, + "docstring": "CSV client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads csv files directly from the disk and\n returns their raw binary contents.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.csv.FileSystemCsvClient.fetch", + "signature": "fetch(path: Path)", + "docstring": "Read a csv file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the csv file.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." + } + } + }, + "CsvParser": { + "name": "CsvParser", + "kind": "class", + "path": "omniread.csv.CsvParser", + "signature": "CsvParser(content: Content)", + "docstring": "Generic csv parser producing string rows from the document.\n\nNotes:\n **Responsibilities:**\n\n - Decode the payload (UTF-8 with BOM support, Latin-1 fallback).\n - Detect the delimiter from a leading sample (`,` `;` tab `|`),\n defaulting to `,`.\n - Normalize cells into deterministic stripped string values.\n - Expose row extraction helpers mirroring `XlsxParser.rows`.\n\n **Constraints:**\n\n - All values are strings; consumers requiring typed values must\n convert on their side.\n - Quoted fields containing delimiters/newlines are handled by\n the standard ``csv`` module.", + "members": { + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.csv.CsvParser.parse", + "signature": "parse()", + "docstring": "Parse the document into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the document." + }, + "rows": { + "name": "rows", + "kind": "function", + "path": "omniread.csv.CsvParser.rows", + "signature": "rows(*, skip_empty: bool = True)", + "docstring": "Extract normalized string rows from the document.\n\nArgs:\n skip_empty (bool):\n When True (default), rows whose cells are all blank are\n omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row." + } + } + }, + "CsvParserBase": { + "name": "CsvParserBase", + "kind": "class", + "path": "omniread.csv.CsvParserBase", + "signature": null, + "docstring": "Base csv parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces csv content-type compatibility and provides\n the extension point for implementing concrete csv parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.csv.CsvParserBase.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser (CSV only)." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.csv.CsvParserBase.parse", + "signature": "parse()", + "docstring": "Parse csv content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation." + } + } + }, + "CsvScraper": { + "name": "CsvScraper", + "kind": "class", + "path": "omniread.csv.CsvScraper", + "signature": "CsvScraper(*, client: BaseCsvClient)", + "docstring": "Scraper for csv documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw csv bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n CSV content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.csv.CsvScraper.fetch", + "signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None)", + "docstring": "Fetch a csv document from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw csv bytes, source\n identifier, CSV content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client." + } + } + }, + "client": { + "name": "client", + "kind": "module", + "path": "omniread.csv.client", + "signature": null, + "docstring": "# Summary\n\nCSV client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw\ncomma-separated-value document bytes from a concrete backing store.\n\nClients provide low-level access to csv binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems", + "members": { + "BaseCsvClient": { + "name": "BaseCsvClient", + "kind": "class", + "path": "omniread.csv.client.BaseCsvClient", + "signature": null, + "docstring": "Abstract client responsible for retrieving csv bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full csv binary payload.\n - Raise retrieval-specific errors on failure.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.csv.client.BaseCsvClient.fetch", + "signature": "fetch(source: Any) -> bytes", + "docstring": "Fetch raw csv bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." + } + } + }, + "FileSystemCsvClient": { + "name": "FileSystemCsvClient", + "kind": "class", + "path": "omniread.csv.client.FileSystemCsvClient", + "signature": null, + "docstring": "CSV client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads csv files directly from the disk and\n returns their raw binary contents.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.csv.client.FileSystemCsvClient.fetch", + "signature": "fetch(path: Path) -> bytes", + "docstring": "Read a csv file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the csv file.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." + } + } + } + } + }, + "parser": { + "name": "parser", + "kind": "module", + "path": "omniread.csv.parser", + "signature": null, + "docstring": "# Summary\n\nCSV parser implementations for OmniRead.\n\nThis module provides a concrete, generic parser for comma-separated-value\ndocuments. It exposes records as lists of string cells so downstream\nconsumers can interpret tabular content without depending on the ``csv``\nmodule directly.\n\nThe parser is intentionally statement-agnostic: it performs no header\ndetection or column interpretation beyond basic cell normalization and\ndelimiter detection.", + "members": { + "Content": { + "name": "Content", + "kind": "class", + "path": "omniread.csv.parser.Content", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", + "members": { + "raw": { + "name": "raw", + "kind": "attribute", + "path": "omniread.csv.parser.Content.raw", + "signature": null, + "docstring": "Raw content bytes as retrieved from the source." + }, + "source": { + "name": "source", + "kind": "attribute", + "path": "omniread.csv.parser.Content.source", + "signature": null, + "docstring": "Identifier of the content origin (URL, file path, or logical name)." + }, + "content_type": { + "name": "content_type", + "kind": "attribute", + "path": "omniread.csv.parser.Content.content_type", + "signature": null, + "docstring": "Optional MIME type of the content, if known." + }, + "metadata": { + "name": "metadata", + "kind": "attribute", + "path": "omniread.csv.parser.Content.metadata", + "signature": null, + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." + } + } + }, + "CsvParserBase": { + "name": "CsvParserBase", + "kind": "class", + "path": "omniread.csv.parser.CsvParserBase", + "signature": null, + "docstring": "Base csv parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces csv content-type compatibility and provides\n the extension point for implementing concrete csv parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.csv.parser.CsvParserBase.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser (CSV only)." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.csv.parser.CsvParserBase.parse", + "signature": "parse()", + "docstring": "Parse csv content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation." + } + } + }, + "CsvParser": { + "name": "CsvParser", + "kind": "class", + "path": "omniread.csv.parser.CsvParser", + "signature": "CsvParser(content: Content)", + "docstring": "Generic csv parser producing string rows from the document.\n\nNotes:\n **Responsibilities:**\n\n - Decode the payload (UTF-8 with BOM support, Latin-1 fallback).\n - Detect the delimiter from a leading sample (`,` `;` tab `|`),\n defaulting to `,`.\n - Normalize cells into deterministic stripped string values.\n - Expose row extraction helpers mirroring `XlsxParser.rows`.\n\n **Constraints:**\n\n - All values are strings; consumers requiring typed values must\n convert on their side.\n - Quoted fields containing delimiters/newlines are handled by\n the standard ``csv`` module.", + "members": { + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.csv.parser.CsvParser.parse", + "signature": "parse() -> list[list[str]]", + "docstring": "Parse the document into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the document." + }, + "rows": { + "name": "rows", + "kind": "function", + "path": "omniread.csv.parser.CsvParser.rows", + "signature": "rows(*, skip_empty: bool = True) -> list[list[str]]", + "docstring": "Extract normalized string rows from the document.\n\nArgs:\n skip_empty (bool):\n When True (default), rows whose cells are all blank are\n omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row." + } + } + } + } + }, + "parser_base": { + "name": "parser_base", + "kind": "module", + "path": "omniread.csv.parser_base", + "signature": null, + "docstring": "# Summary\n\nCSV parser base implementation for OmniRead.\n\nThis module defines the **CSV-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for\ncomma-separated-value documents.", + "members": { + "ContentType": { + "name": "ContentType", + "kind": "class", + "path": "omniread.csv.parser_base.ContentType", + "signature": null, + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", + "members": { + "HTML": { + "name": "HTML", + "kind": "attribute", + "path": "omniread.csv.parser_base.ContentType.HTML", + "signature": null, + "docstring": "HTML document content." + }, + "PDF": { + "name": "PDF", + "kind": "attribute", + "path": "omniread.csv.parser_base.ContentType.PDF", + "signature": null, + "docstring": "PDF document content." + }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.csv.parser_base.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.csv.parser_base.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, + "JSON": { + "name": "JSON", + "kind": "attribute", + "path": "omniread.csv.parser_base.ContentType.JSON", + "signature": null, + "docstring": "JSON document content." + }, + "XML": { + "name": "XML", + "kind": "attribute", + "path": "omniread.csv.parser_base.ContentType.XML", + "signature": null, + "docstring": "XML document content." + } + } + }, + "BaseParser": { + "name": "BaseParser", + "kind": "class", + "path": "omniread.csv.parser_base.BaseParser", + "signature": "BaseParser(content: Content)", + "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.csv.parser_base.BaseParser.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." + }, + "content": { + "name": "content", + "kind": "attribute", + "path": "omniread.csv.parser_base.BaseParser.content", + "signature": null, + "docstring": null + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.csv.parser_base.BaseParser.parse", + "signature": "parse()", + "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." + }, + "supports": { + "name": "supports", + "kind": "function", + "path": "omniread.csv.parser_base.BaseParser.supports", + "signature": "supports()", + "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." + } + } + }, + "T": { + "name": "T", + "kind": "attribute", + "path": "omniread.csv.parser_base.T", + "signature": null, + "docstring": null + }, + "CsvParserBase": { + "name": "CsvParserBase", + "kind": "class", + "path": "omniread.csv.parser_base.CsvParserBase", + "signature": null, + "docstring": "Base csv parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces csv content-type compatibility and provides\n the extension point for implementing concrete csv parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.csv.parser_base.CsvParserBase.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser (CSV only)." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.csv.parser_base.CsvParserBase.parse", + "signature": "parse() -> T", + "docstring": "Parse csv content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation." + } + } + } + } + }, + "scraper": { + "name": "scraper", + "kind": "module", + "path": "omniread.csv.scraper", + "signature": null, + "docstring": "# Summary\n\nCSV scraper for OmniRead.\n\nThis module defines the scraper responsible for acquiring raw\ncomma-separated-value document content from a backing store via a\nconfigured client.\n\nThe scraper does not interpret or parse the acquired bytes; it wraps them in\nthe canonical `Content` model.", + "members": { + "Content": { + "name": "Content", + "kind": "class", + "path": "omniread.csv.scraper.Content", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", + "members": { + "raw": { + "name": "raw", + "kind": "attribute", + "path": "omniread.csv.scraper.Content.raw", + "signature": null, + "docstring": "Raw content bytes as retrieved from the source." + }, + "source": { + "name": "source", + "kind": "attribute", + "path": "omniread.csv.scraper.Content.source", + "signature": null, + "docstring": "Identifier of the content origin (URL, file path, or logical name)." + }, + "content_type": { + "name": "content_type", + "kind": "attribute", + "path": "omniread.csv.scraper.Content.content_type", + "signature": null, + "docstring": "Optional MIME type of the content, if known." + }, + "metadata": { + "name": "metadata", + "kind": "attribute", + "path": "omniread.csv.scraper.Content.metadata", + "signature": null, + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." + } + } + }, + "ContentType": { + "name": "ContentType", + "kind": "class", + "path": "omniread.csv.scraper.ContentType", + "signature": null, + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", + "members": { + "HTML": { + "name": "HTML", + "kind": "attribute", + "path": "omniread.csv.scraper.ContentType.HTML", + "signature": null, + "docstring": "HTML document content." + }, + "PDF": { + "name": "PDF", + "kind": "attribute", + "path": "omniread.csv.scraper.ContentType.PDF", + "signature": null, + "docstring": "PDF document content." + }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.csv.scraper.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.csv.scraper.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, + "JSON": { + "name": "JSON", + "kind": "attribute", + "path": "omniread.csv.scraper.ContentType.JSON", + "signature": null, + "docstring": "JSON document content." + }, + "XML": { + "name": "XML", + "kind": "attribute", + "path": "omniread.csv.scraper.ContentType.XML", + "signature": null, + "docstring": "XML document content." + } + } + }, + "BaseCsvClient": { + "name": "BaseCsvClient", + "kind": "class", + "path": "omniread.csv.scraper.BaseCsvClient", + "signature": null, + "docstring": "Abstract client responsible for retrieving csv bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full csv binary payload.\n - Raise retrieval-specific errors on failure.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.csv.scraper.BaseCsvClient.fetch", + "signature": "fetch(source: Any)", + "docstring": "Fetch raw csv bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." + } + } + }, + "CsvScraper": { + "name": "CsvScraper", + "kind": "class", + "path": "omniread.csv.scraper.CsvScraper", + "signature": "CsvScraper(*, client: BaseCsvClient)", + "docstring": "Scraper for csv documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw csv bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n CSV content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.csv.scraper.CsvScraper.fetch", + "signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None) -> Content", + "docstring": "Fetch a csv document from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw csv bytes, source\n identifier, CSV content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client." + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/docs/mcp/modules/omniread.csv.parser.json b/docs/mcp/modules/omniread.csv.parser.json new file mode 100644 index 0000000..50a0fe5 --- /dev/null +++ b/docs/mcp/modules/omniread.csv.parser.json @@ -0,0 +1,92 @@ +{ + "module": "omniread.csv.parser", + "content": { + "path": "omniread.csv.parser", + "docstring": "# Summary\n\nCSV parser implementations for OmniRead.\n\nThis module provides a concrete, generic parser for comma-separated-value\ndocuments. It exposes records as lists of string cells so downstream\nconsumers can interpret tabular content without depending on the ``csv``\nmodule directly.\n\nThe parser is intentionally statement-agnostic: it performs no header\ndetection or column interpretation beyond basic cell normalization and\ndelimiter detection.", + "objects": { + "Content": { + "name": "Content", + "kind": "class", + "path": "omniread.csv.parser.Content", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", + "members": { + "raw": { + "name": "raw", + "kind": "attribute", + "path": "omniread.csv.parser.Content.raw", + "signature": null, + "docstring": "Raw content bytes as retrieved from the source." + }, + "source": { + "name": "source", + "kind": "attribute", + "path": "omniread.csv.parser.Content.source", + "signature": null, + "docstring": "Identifier of the content origin (URL, file path, or logical name)." + }, + "content_type": { + "name": "content_type", + "kind": "attribute", + "path": "omniread.csv.parser.Content.content_type", + "signature": null, + "docstring": "Optional MIME type of the content, if known." + }, + "metadata": { + "name": "metadata", + "kind": "attribute", + "path": "omniread.csv.parser.Content.metadata", + "signature": null, + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." + } + } + }, + "CsvParserBase": { + "name": "CsvParserBase", + "kind": "class", + "path": "omniread.csv.parser.CsvParserBase", + "signature": null, + "docstring": "Base csv parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces csv content-type compatibility and provides\n the extension point for implementing concrete csv parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.csv.parser.CsvParserBase.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser (CSV only)." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.csv.parser.CsvParserBase.parse", + "signature": "parse()", + "docstring": "Parse csv content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation." + } + } + }, + "CsvParser": { + "name": "CsvParser", + "kind": "class", + "path": "omniread.csv.parser.CsvParser", + "signature": "CsvParser(content: Content)", + "docstring": "Generic csv parser producing string rows from the document.\n\nNotes:\n **Responsibilities:**\n\n - Decode the payload (UTF-8 with BOM support, Latin-1 fallback).\n - Detect the delimiter from a leading sample (`,` `;` tab `|`),\n defaulting to `,`.\n - Normalize cells into deterministic stripped string values.\n - Expose row extraction helpers mirroring `XlsxParser.rows`.\n\n **Constraints:**\n\n - All values are strings; consumers requiring typed values must\n convert on their side.\n - Quoted fields containing delimiters/newlines are handled by\n the standard ``csv`` module.", + "members": { + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.csv.parser.CsvParser.parse", + "signature": "parse() -> list[list[str]]", + "docstring": "Parse the document into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the document." + }, + "rows": { + "name": "rows", + "kind": "function", + "path": "omniread.csv.parser.CsvParser.rows", + "signature": "rows(*, skip_empty: bool = True) -> list[list[str]]", + "docstring": "Extract normalized string rows from the document.\n\nArgs:\n skip_empty (bool):\n When True (default), rows whose cells are all blank are\n omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row." + } + } + } + } + } +} \ No newline at end of file diff --git a/docs/mcp/modules/omniread.csv.parser_base.json b/docs/mcp/modules/omniread.csv.parser_base.json new file mode 100644 index 0000000..4c15183 --- /dev/null +++ b/docs/mcp/modules/omniread.csv.parser_base.json @@ -0,0 +1,127 @@ +{ + "module": "omniread.csv.parser_base", + "content": { + "path": "omniread.csv.parser_base", + "docstring": "# Summary\n\nCSV parser base implementation for OmniRead.\n\nThis module defines the **CSV-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for\ncomma-separated-value documents.", + "objects": { + "ContentType": { + "name": "ContentType", + "kind": "class", + "path": "omniread.csv.parser_base.ContentType", + "signature": null, + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", + "members": { + "HTML": { + "name": "HTML", + "kind": "attribute", + "path": "omniread.csv.parser_base.ContentType.HTML", + "signature": null, + "docstring": "HTML document content." + }, + "PDF": { + "name": "PDF", + "kind": "attribute", + "path": "omniread.csv.parser_base.ContentType.PDF", + "signature": null, + "docstring": "PDF document content." + }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.csv.parser_base.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.csv.parser_base.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, + "JSON": { + "name": "JSON", + "kind": "attribute", + "path": "omniread.csv.parser_base.ContentType.JSON", + "signature": null, + "docstring": "JSON document content." + }, + "XML": { + "name": "XML", + "kind": "attribute", + "path": "omniread.csv.parser_base.ContentType.XML", + "signature": null, + "docstring": "XML document content." + } + } + }, + "BaseParser": { + "name": "BaseParser", + "kind": "class", + "path": "omniread.csv.parser_base.BaseParser", + "signature": "BaseParser(content: Content)", + "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.csv.parser_base.BaseParser.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." + }, + "content": { + "name": "content", + "kind": "attribute", + "path": "omniread.csv.parser_base.BaseParser.content", + "signature": null, + "docstring": null + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.csv.parser_base.BaseParser.parse", + "signature": "parse()", + "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." + }, + "supports": { + "name": "supports", + "kind": "function", + "path": "omniread.csv.parser_base.BaseParser.supports", + "signature": "supports()", + "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." + } + } + }, + "T": { + "name": "T", + "kind": "attribute", + "path": "omniread.csv.parser_base.T", + "signature": null, + "docstring": null + }, + "CsvParserBase": { + "name": "CsvParserBase", + "kind": "class", + "path": "omniread.csv.parser_base.CsvParserBase", + "signature": null, + "docstring": "Base csv parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces csv content-type compatibility and provides\n the extension point for implementing concrete csv parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.csv.parser_base.CsvParserBase.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser (CSV only)." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.csv.parser_base.CsvParserBase.parse", + "signature": "parse() -> T", + "docstring": "Parse csv content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation." + } + } + } + } + } +} \ No newline at end of file diff --git a/docs/mcp/modules/omniread.csv.scraper.json b/docs/mcp/modules/omniread.csv.scraper.json new file mode 100644 index 0000000..be51654 --- /dev/null +++ b/docs/mcp/modules/omniread.csv.scraper.json @@ -0,0 +1,129 @@ +{ + "module": "omniread.csv.scraper", + "content": { + "path": "omniread.csv.scraper", + "docstring": "# Summary\n\nCSV scraper for OmniRead.\n\nThis module defines the scraper responsible for acquiring raw\ncomma-separated-value document content from a backing store via a\nconfigured client.\n\nThe scraper does not interpret or parse the acquired bytes; it wraps them in\nthe canonical `Content` model.", + "objects": { + "Content": { + "name": "Content", + "kind": "class", + "path": "omniread.csv.scraper.Content", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", + "members": { + "raw": { + "name": "raw", + "kind": "attribute", + "path": "omniread.csv.scraper.Content.raw", + "signature": null, + "docstring": "Raw content bytes as retrieved from the source." + }, + "source": { + "name": "source", + "kind": "attribute", + "path": "omniread.csv.scraper.Content.source", + "signature": null, + "docstring": "Identifier of the content origin (URL, file path, or logical name)." + }, + "content_type": { + "name": "content_type", + "kind": "attribute", + "path": "omniread.csv.scraper.Content.content_type", + "signature": null, + "docstring": "Optional MIME type of the content, if known." + }, + "metadata": { + "name": "metadata", + "kind": "attribute", + "path": "omniread.csv.scraper.Content.metadata", + "signature": null, + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." + } + } + }, + "ContentType": { + "name": "ContentType", + "kind": "class", + "path": "omniread.csv.scraper.ContentType", + "signature": null, + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", + "members": { + "HTML": { + "name": "HTML", + "kind": "attribute", + "path": "omniread.csv.scraper.ContentType.HTML", + "signature": null, + "docstring": "HTML document content." + }, + "PDF": { + "name": "PDF", + "kind": "attribute", + "path": "omniread.csv.scraper.ContentType.PDF", + "signature": null, + "docstring": "PDF document content." + }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.csv.scraper.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.csv.scraper.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, + "JSON": { + "name": "JSON", + "kind": "attribute", + "path": "omniread.csv.scraper.ContentType.JSON", + "signature": null, + "docstring": "JSON document content." + }, + "XML": { + "name": "XML", + "kind": "attribute", + "path": "omniread.csv.scraper.ContentType.XML", + "signature": null, + "docstring": "XML document content." + } + } + }, + "BaseCsvClient": { + "name": "BaseCsvClient", + "kind": "class", + "path": "omniread.csv.scraper.BaseCsvClient", + "signature": null, + "docstring": "Abstract client responsible for retrieving csv bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full csv binary payload.\n - Raise retrieval-specific errors on failure.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.csv.scraper.BaseCsvClient.fetch", + "signature": "fetch(source: Any)", + "docstring": "Fetch raw csv bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." + } + } + }, + "CsvScraper": { + "name": "CsvScraper", + "kind": "class", + "path": "omniread.csv.scraper.CsvScraper", + "signature": "CsvScraper(*, client: BaseCsvClient)", + "docstring": "Scraper for csv documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw csv bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n CSV content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.csv.scraper.CsvScraper.fetch", + "signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None) -> Content", + "docstring": "Fetch a csv document from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw csv bytes, source\n identifier, CSV content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client." + } + } + } + } + } +} \ No newline at end of file diff --git a/mcp_docs/modules/omniread.html.json b/docs/mcp/modules/omniread.html.json similarity index 67% rename from mcp_docs/modules/omniread.html.json rename to docs/mcp/modules/omniread.html.json index e079b2d..3ffb5b5 100644 --- a/mcp_docs/modules/omniread.html.json +++ b/docs/mcp/modules/omniread.html.json @@ -8,29 +8,29 @@ "name": "HTMLScraper", "kind": "class", "path": "omniread.html.HTMLScraper", - "signature": "", + "signature": "HTMLScraper(*, client: httpx.Client | None = None, timeout: float = 15.0, headers: Mapping[str, str] | None = None, follow_redirects: bool = True)", "docstring": "Base HTML scraper using `httpx`.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns\n them as raw content wrapped in a `Content` object.\n - Fetches raw bytes and metadata only.\n - The scraper uses `httpx.Client` for HTTP requests, enforces an\n HTML content type, and preserves HTTP response metadata.\n\n **Constraints:**\n\n - The scraper does not: Parse HTML, perform retries or backoff,\n handle non-HTML responses.", "members": { "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.html.HTMLScraper.content_type", - "signature": "", + "signature": null, "docstring": null }, "validate_content_type": { "name": "validate_content_type", "kind": "function", "path": "omniread.html.HTMLScraper.validate_content_type", - "signature": "", + "signature": "validate_content_type(response: httpx.Response)", "docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content." }, "fetch": { "name": "fetch", "kind": "function", "path": "omniread.html.HTMLScraper.fetch", - "signature": "", - "docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML." + "signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None)", + "docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML." } } }, @@ -38,49 +38,49 @@ "name": "HTMLParser", "kind": "class", "path": "omniread.html.HTMLParser", - "signature": "", + "signature": "HTMLParser(content: Content, features: str = 'html.parser')", "docstring": "Base HTML parser.\n\nNotes:\n **Responsibilities:**\n\n - This class extends the core `BaseParser` with HTML-specific behavior,\n including DOM parsing via BeautifulSoup and reusable extraction helpers.\n - Provides reusable helpers for HTML extraction. Concrete parsers must\n explicitly define the return type.\n\n **Guarantees:**\n\n - Accepts only HTML content.\n - Owns a parsed BeautifulSoup DOM tree.\n - Provides pure helper utilities for common HTML structures.\n\n **Constraints:**\n\n - Concrete subclasses must define the output type `T` and implement\n the `parse()` method.", "members": { "supported_types": { "name": "supported_types", "kind": "attribute", "path": "omniread.html.HTMLParser.supported_types", - "signature": "", + "signature": null, "docstring": "Set of content types supported by this parser (HTML only)." }, "parse": { "name": "parse", "kind": "function", "path": "omniread.html.HTMLParser.parse", - "signature": "", + "signature": "parse()", "docstring": "Fully parse the HTML content into structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the HTML DOM and return a\n deterministic, structured output." }, "parse_div": { "name": "parse_div", "kind": "function", "path": "omniread.html.HTMLParser.parse_div", - "signature": "", + "signature": "parse_div(div: Tag, *, separator: str = ' ')", "docstring": "Extract normalized text from a `
` element.\n\nArgs:\n div (Tag):\n BeautifulSoup tag representing a `
`.\n separator (str, optional):\n String used to separate text nodes.\n\nReturns:\n str:\n Flattened, whitespace-normalized text content." }, "parse_link": { "name": "parse_link", "kind": "function", "path": "omniread.html.HTMLParser.parse_link", - "signature": "", - "docstring": "Extract the hyperlink reference from an `` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n Optional[str]:\n The value of the `href` attribute, or None if absent." + "signature": "parse_link(a: Tag)", + "docstring": "Extract the hyperlink reference from an `` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n str | None:\n The value of the `href` attribute, or None if absent." }, "parse_table": { "name": "parse_table", "kind": "function", "path": "omniread.html.HTMLParser.parse_table", - "signature": "", + "signature": "parse_table(table: Tag)", "docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table (Tag):\n BeautifulSoup tag representing a ``.\n\nReturns:\n list[list[str]]:\n A list of rows, where each row is a list of cell text values." }, "parse_meta": { "name": "parse_meta", "kind": "function", "path": "omniread.html.HTMLParser.parse_meta", - "signature": "", + "signature": "parse_meta()", "docstring": "Extract high-level metadata from the HTML document.\n\nReturns:\n dict[str, Any]:\n Dictionary containing extracted metadata.\n\nNotes:\n **Responsibilities:**\n\n - Extract high-level metadata from the HTML document.\n - This includes: Document title, `` tag name/property to\n content mappings." } } @@ -92,162 +92,127 @@ "signature": null, "docstring": "# Summary\n\nHTML parser base implementations for OmniRead.\n\nThis module provides reusable HTML parsing utilities built on top of\nthe abstract parser contracts defined in `omniread.core.parser`.\n\nIt supplies:\n\n- Content-type enforcement for HTML inputs\n- BeautifulSoup initialization and lifecycle management\n- Common helper methods for extracting structured data from HTML elements\n\nConcrete parsers must subclass `HTMLParser` and implement the `parse()` method\nto return a structured representation appropriate for their use case.", "members": { - "Any": { - "name": "Any", - "kind": "alias", - "path": "omniread.html.parser.Any", - "signature": "", - "docstring": null - }, - "Generic": { - "name": "Generic", - "kind": "alias", - "path": "omniread.html.parser.Generic", - "signature": "", - "docstring": null - }, - "TypeVar": { - "name": "TypeVar", - "kind": "alias", - "path": "omniread.html.parser.TypeVar", - "signature": "", - "docstring": null - }, - "Optional": { - "name": "Optional", - "kind": "alias", - "path": "omniread.html.parser.Optional", - "signature": "", - "docstring": null - }, - "abstractmethod": { - "name": "abstractmethod", - "kind": "alias", - "path": "omniread.html.parser.abstractmethod", - "signature": "", - "docstring": null - }, - "BeautifulSoup": { - "name": "BeautifulSoup", - "kind": "alias", - "path": "omniread.html.parser.BeautifulSoup", - "signature": "", - "docstring": null - }, - "Tag": { - "name": "Tag", - "kind": "alias", - "path": "omniread.html.parser.Tag", - "signature": "", - "docstring": null - }, - "ContentType": { - "name": "ContentType", - "kind": "class", - "path": "omniread.html.parser.ContentType", - "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", - "members": { - "HTML": { - "name": "HTML", - "kind": "attribute", - "path": "omniread.html.parser.ContentType.HTML", - "signature": "", - "docstring": "HTML document content." - }, - "PDF": { - "name": "PDF", - "kind": "attribute", - "path": "omniread.html.parser.ContentType.PDF", - "signature": "", - "docstring": "PDF document content." - }, - "JSON": { - "name": "JSON", - "kind": "attribute", - "path": "omniread.html.parser.ContentType.JSON", - "signature": "", - "docstring": "JSON document content." - }, - "XML": { - "name": "XML", - "kind": "attribute", - "path": "omniread.html.parser.ContentType.XML", - "signature": "", - "docstring": "XML document content." - } - } - }, "Content": { "name": "Content", "kind": "class", "path": "omniread.html.parser.Content", - "signature": "", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.html.parser.Content.raw", - "signature": "", + "signature": null, "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.html.parser.Content.source", - "signature": "", + "signature": null, "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.html.parser.Content.content_type", - "signature": "", + "signature": null, "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.html.parser.Content.metadata", - "signature": "", + "signature": null, "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } }, + "ContentType": { + "name": "ContentType", + "kind": "class", + "path": "omniread.html.parser.ContentType", + "signature": null, + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", + "members": { + "HTML": { + "name": "HTML", + "kind": "attribute", + "path": "omniread.html.parser.ContentType.HTML", + "signature": null, + "docstring": "HTML document content." + }, + "PDF": { + "name": "PDF", + "kind": "attribute", + "path": "omniread.html.parser.ContentType.PDF", + "signature": null, + "docstring": "PDF document content." + }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.html.parser.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.html.parser.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, + "JSON": { + "name": "JSON", + "kind": "attribute", + "path": "omniread.html.parser.ContentType.JSON", + "signature": null, + "docstring": "JSON document content." + }, + "XML": { + "name": "XML", + "kind": "attribute", + "path": "omniread.html.parser.ContentType.XML", + "signature": null, + "docstring": "XML document content." + } + } + }, "BaseParser": { "name": "BaseParser", "kind": "class", "path": "omniread.html.parser.BaseParser", - "signature": "", + "signature": "BaseParser(content: Content)", "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", "members": { "supported_types": { "name": "supported_types", "kind": "attribute", "path": "omniread.html.parser.BaseParser.supported_types", - "signature": "", + "signature": null, "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." }, "content": { "name": "content", "kind": "attribute", "path": "omniread.html.parser.BaseParser.content", - "signature": "", + "signature": null, "docstring": null }, "parse": { "name": "parse", "kind": "function", "path": "omniread.html.parser.BaseParser.parse", - "signature": "", + "signature": "parse()", "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." }, "supports": { "name": "supports", "kind": "function", "path": "omniread.html.parser.BaseParser.supports", - "signature": "", + "signature": "supports()", "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." } } @@ -263,7 +228,7 @@ "name": "HTMLParser", "kind": "class", "path": "omniread.html.parser.HTMLParser", - "signature": "", + "signature": "HTMLParser(content: Content, features: str = 'html.parser')", "docstring": "Base HTML parser.\n\nNotes:\n **Responsibilities:**\n\n - This class extends the core `BaseParser` with HTML-specific behavior,\n including DOM parsing via BeautifulSoup and reusable extraction helpers.\n - Provides reusable helpers for HTML extraction. Concrete parsers must\n explicitly define the return type.\n\n **Guarantees:**\n\n - Accepts only HTML content.\n - Owns a parsed BeautifulSoup DOM tree.\n - Provides pure helper utilities for common HTML structures.\n\n **Constraints:**\n\n - Concrete subclasses must define the output type `T` and implement\n the `parse()` method.", "members": { "supported_types": { @@ -277,52 +242,38 @@ "name": "parse", "kind": "function", "path": "omniread.html.parser.HTMLParser.parse", - "signature": "", + "signature": "parse() -> T", "docstring": "Fully parse the HTML content into structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the HTML DOM and return a\n deterministic, structured output." }, "parse_div": { "name": "parse_div", "kind": "function", "path": "omniread.html.parser.HTMLParser.parse_div", - "signature": "", + "signature": "parse_div(div: Tag, *, separator: str = ' ') -> str", "docstring": "Extract normalized text from a `
` element.\n\nArgs:\n div (Tag):\n BeautifulSoup tag representing a `
`.\n separator (str, optional):\n String used to separate text nodes.\n\nReturns:\n str:\n Flattened, whitespace-normalized text content." }, "parse_link": { "name": "parse_link", "kind": "function", "path": "omniread.html.parser.HTMLParser.parse_link", - "signature": "", - "docstring": "Extract the hyperlink reference from an `` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n Optional[str]:\n The value of the `href` attribute, or None if absent." + "signature": "parse_link(a: Tag) -> str | None", + "docstring": "Extract the hyperlink reference from an `` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n str | None:\n The value of the `href` attribute, or None if absent." }, "parse_table": { "name": "parse_table", "kind": "function", "path": "omniread.html.parser.HTMLParser.parse_table", - "signature": "", + "signature": "parse_table(table: Tag) -> list[list[str]]", "docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table (Tag):\n BeautifulSoup tag representing a `
`.\n\nReturns:\n list[list[str]]:\n A list of rows, where each row is a list of cell text values." }, "parse_meta": { "name": "parse_meta", "kind": "function", "path": "omniread.html.parser.HTMLParser.parse_meta", - "signature": "", + "signature": "parse_meta() -> dict[str, Any]", "docstring": "Extract high-level metadata from the HTML document.\n\nReturns:\n dict[str, Any]:\n Dictionary containing extracted metadata.\n\nNotes:\n **Responsibilities:**\n\n - Extract high-level metadata from the HTML document.\n - This includes: Document title, `` tag name/property to\n content mappings." } } - }, - "list": { - "name": "list", - "kind": "alias", - "path": "omniread.html.parser.list", - "signature": "", - "docstring": null - }, - "dict": { - "name": "dict", - "kind": "alias", - "path": "omniread.html.parser.dict", - "signature": "", - "docstring": null } } }, @@ -333,67 +284,39 @@ "signature": null, "docstring": "# Summary\n\nHTML scraping implementation for OmniRead.\n\nThis module provides an HTTP-based scraper for retrieving HTML documents.\nIt implements the core `BaseScraper` contract using `httpx` as the transport\nlayer.\n\nThis scraper is responsible for:\n\n- Fetching raw HTML bytes over HTTP(S)\n- Validating response content type\n- Attaching HTTP metadata to the returned content\n\nThis scraper is not responsible for:\n\n- Parsing or interpreting HTML\n- Retrying failed requests\n- Managing crawl policies or rate limiting", "members": { - "httpx": { - "name": "httpx", - "kind": "alias", - "path": "omniread.html.scraper.httpx", - "signature": "", - "docstring": null - }, - "Any": { - "name": "Any", - "kind": "alias", - "path": "omniread.html.scraper.Any", - "signature": "", - "docstring": null - }, - "Mapping": { - "name": "Mapping", - "kind": "alias", - "path": "omniread.html.scraper.Mapping", - "signature": "", - "docstring": null - }, - "Optional": { - "name": "Optional", - "kind": "alias", - "path": "omniread.html.scraper.Optional", - "signature": "", - "docstring": null - }, "Content": { "name": "Content", "kind": "class", "path": "omniread.html.scraper.Content", - "signature": "", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.html.scraper.Content.raw", - "signature": "", + "signature": null, "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.html.scraper.Content.source", - "signature": "", + "signature": null, "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.html.scraper.Content.content_type", - "signature": "", + "signature": null, "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.html.scraper.Content.metadata", - "signature": "", + "signature": null, "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } @@ -402,35 +325,49 @@ "name": "ContentType", "kind": "class", "path": "omniread.html.scraper.ContentType", - "signature": "", + "signature": null, "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", "kind": "attribute", "path": "omniread.html.scraper.ContentType.HTML", - "signature": "", + "signature": null, "docstring": "HTML document content." }, "PDF": { "name": "PDF", "kind": "attribute", "path": "omniread.html.scraper.ContentType.PDF", - "signature": "", + "signature": null, "docstring": "PDF document content." }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.html.scraper.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.html.scraper.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, "JSON": { "name": "JSON", "kind": "attribute", "path": "omniread.html.scraper.ContentType.JSON", - "signature": "", + "signature": null, "docstring": "JSON document content." }, "XML": { "name": "XML", "kind": "attribute", "path": "omniread.html.scraper.ContentType.XML", - "signature": "", + "signature": null, "docstring": "XML document content." } } @@ -439,15 +376,15 @@ "name": "BaseScraper", "kind": "class", "path": "omniread.html.scraper.BaseScraper", - "signature": "", + "signature": null, "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.html.scraper.BaseScraper.fetch", - "signature": "", - "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." + "signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None)", + "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." } } }, @@ -455,7 +392,7 @@ "name": "HTMLScraper", "kind": "class", "path": "omniread.html.scraper.HTMLScraper", - "signature": "", + "signature": "HTMLScraper(*, client: httpx.Client | None = None, timeout: float = 15.0, headers: Mapping[str, str] | None = None, follow_redirects: bool = True)", "docstring": "Base HTML scraper using `httpx`.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns\n them as raw content wrapped in a `Content` object.\n - Fetches raw bytes and metadata only.\n - The scraper uses `httpx.Client` for HTTP requests, enforces an\n HTML content type, and preserves HTTP response metadata.\n\n **Constraints:**\n\n - The scraper does not: Parse HTML, perform retries or backoff,\n handle non-HTML responses.", "members": { "content_type": { @@ -469,15 +406,15 @@ "name": "validate_content_type", "kind": "function", "path": "omniread.html.scraper.HTMLScraper.validate_content_type", - "signature": "", + "signature": "validate_content_type(response: httpx.Response) -> None", "docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content." }, "fetch": { "name": "fetch", "kind": "function", "path": "omniread.html.scraper.HTMLScraper.fetch", - "signature": "", - "docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML." + "signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None) -> Content", + "docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML." } } } diff --git a/mcp_docs/modules/omniread.html.parser.json b/docs/mcp/modules/omniread.html.parser.json similarity index 66% rename from mcp_docs/modules/omniread.html.parser.json rename to docs/mcp/modules/omniread.html.parser.json index c607d2d..31c9c59 100644 --- a/mcp_docs/modules/omniread.html.parser.json +++ b/docs/mcp/modules/omniread.html.parser.json @@ -4,162 +4,127 @@ "path": "omniread.html.parser", "docstring": "# Summary\n\nHTML parser base implementations for OmniRead.\n\nThis module provides reusable HTML parsing utilities built on top of\nthe abstract parser contracts defined in `omniread.core.parser`.\n\nIt supplies:\n\n- Content-type enforcement for HTML inputs\n- BeautifulSoup initialization and lifecycle management\n- Common helper methods for extracting structured data from HTML elements\n\nConcrete parsers must subclass `HTMLParser` and implement the `parse()` method\nto return a structured representation appropriate for their use case.", "objects": { - "Any": { - "name": "Any", - "kind": "alias", - "path": "omniread.html.parser.Any", - "signature": "", - "docstring": null - }, - "Generic": { - "name": "Generic", - "kind": "alias", - "path": "omniread.html.parser.Generic", - "signature": "", - "docstring": null - }, - "TypeVar": { - "name": "TypeVar", - "kind": "alias", - "path": "omniread.html.parser.TypeVar", - "signature": "", - "docstring": null - }, - "Optional": { - "name": "Optional", - "kind": "alias", - "path": "omniread.html.parser.Optional", - "signature": "", - "docstring": null - }, - "abstractmethod": { - "name": "abstractmethod", - "kind": "alias", - "path": "omniread.html.parser.abstractmethod", - "signature": "", - "docstring": null - }, - "BeautifulSoup": { - "name": "BeautifulSoup", - "kind": "alias", - "path": "omniread.html.parser.BeautifulSoup", - "signature": "", - "docstring": null - }, - "Tag": { - "name": "Tag", - "kind": "alias", - "path": "omniread.html.parser.Tag", - "signature": "", - "docstring": null - }, - "ContentType": { - "name": "ContentType", - "kind": "class", - "path": "omniread.html.parser.ContentType", - "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", - "members": { - "HTML": { - "name": "HTML", - "kind": "attribute", - "path": "omniread.html.parser.ContentType.HTML", - "signature": "", - "docstring": "HTML document content." - }, - "PDF": { - "name": "PDF", - "kind": "attribute", - "path": "omniread.html.parser.ContentType.PDF", - "signature": "", - "docstring": "PDF document content." - }, - "JSON": { - "name": "JSON", - "kind": "attribute", - "path": "omniread.html.parser.ContentType.JSON", - "signature": "", - "docstring": "JSON document content." - }, - "XML": { - "name": "XML", - "kind": "attribute", - "path": "omniread.html.parser.ContentType.XML", - "signature": "", - "docstring": "XML document content." - } - } - }, "Content": { "name": "Content", "kind": "class", "path": "omniread.html.parser.Content", - "signature": "", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.html.parser.Content.raw", - "signature": "", + "signature": null, "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.html.parser.Content.source", - "signature": "", + "signature": null, "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.html.parser.Content.content_type", - "signature": "", + "signature": null, "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.html.parser.Content.metadata", - "signature": "", + "signature": null, "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } }, + "ContentType": { + "name": "ContentType", + "kind": "class", + "path": "omniread.html.parser.ContentType", + "signature": null, + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", + "members": { + "HTML": { + "name": "HTML", + "kind": "attribute", + "path": "omniread.html.parser.ContentType.HTML", + "signature": null, + "docstring": "HTML document content." + }, + "PDF": { + "name": "PDF", + "kind": "attribute", + "path": "omniread.html.parser.ContentType.PDF", + "signature": null, + "docstring": "PDF document content." + }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.html.parser.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.html.parser.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, + "JSON": { + "name": "JSON", + "kind": "attribute", + "path": "omniread.html.parser.ContentType.JSON", + "signature": null, + "docstring": "JSON document content." + }, + "XML": { + "name": "XML", + "kind": "attribute", + "path": "omniread.html.parser.ContentType.XML", + "signature": null, + "docstring": "XML document content." + } + } + }, "BaseParser": { "name": "BaseParser", "kind": "class", "path": "omniread.html.parser.BaseParser", - "signature": "", + "signature": "BaseParser(content: Content)", "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", "members": { "supported_types": { "name": "supported_types", "kind": "attribute", "path": "omniread.html.parser.BaseParser.supported_types", - "signature": "", + "signature": null, "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." }, "content": { "name": "content", "kind": "attribute", "path": "omniread.html.parser.BaseParser.content", - "signature": "", + "signature": null, "docstring": null }, "parse": { "name": "parse", "kind": "function", "path": "omniread.html.parser.BaseParser.parse", - "signature": "", + "signature": "parse()", "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." }, "supports": { "name": "supports", "kind": "function", "path": "omniread.html.parser.BaseParser.supports", - "signature": "", + "signature": "supports()", "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." } } @@ -175,7 +140,7 @@ "name": "HTMLParser", "kind": "class", "path": "omniread.html.parser.HTMLParser", - "signature": "", + "signature": "HTMLParser(content: Content, features: str = 'html.parser')", "docstring": "Base HTML parser.\n\nNotes:\n **Responsibilities:**\n\n - This class extends the core `BaseParser` with HTML-specific behavior,\n including DOM parsing via BeautifulSoup and reusable extraction helpers.\n - Provides reusable helpers for HTML extraction. Concrete parsers must\n explicitly define the return type.\n\n **Guarantees:**\n\n - Accepts only HTML content.\n - Owns a parsed BeautifulSoup DOM tree.\n - Provides pure helper utilities for common HTML structures.\n\n **Constraints:**\n\n - Concrete subclasses must define the output type `T` and implement\n the `parse()` method.", "members": { "supported_types": { @@ -189,52 +154,38 @@ "name": "parse", "kind": "function", "path": "omniread.html.parser.HTMLParser.parse", - "signature": "", + "signature": "parse() -> T", "docstring": "Fully parse the HTML content into structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the HTML DOM and return a\n deterministic, structured output." }, "parse_div": { "name": "parse_div", "kind": "function", "path": "omniread.html.parser.HTMLParser.parse_div", - "signature": "", + "signature": "parse_div(div: Tag, *, separator: str = ' ') -> str", "docstring": "Extract normalized text from a `
` element.\n\nArgs:\n div (Tag):\n BeautifulSoup tag representing a `
`.\n separator (str, optional):\n String used to separate text nodes.\n\nReturns:\n str:\n Flattened, whitespace-normalized text content." }, "parse_link": { "name": "parse_link", "kind": "function", "path": "omniread.html.parser.HTMLParser.parse_link", - "signature": "", - "docstring": "Extract the hyperlink reference from an `` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n Optional[str]:\n The value of the `href` attribute, or None if absent." + "signature": "parse_link(a: Tag) -> str | None", + "docstring": "Extract the hyperlink reference from an `` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n str | None:\n The value of the `href` attribute, or None if absent." }, "parse_table": { "name": "parse_table", "kind": "function", "path": "omniread.html.parser.HTMLParser.parse_table", - "signature": "", + "signature": "parse_table(table: Tag) -> list[list[str]]", "docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table (Tag):\n BeautifulSoup tag representing a `
`.\n\nReturns:\n list[list[str]]:\n A list of rows, where each row is a list of cell text values." }, "parse_meta": { "name": "parse_meta", "kind": "function", "path": "omniread.html.parser.HTMLParser.parse_meta", - "signature": "", + "signature": "parse_meta() -> dict[str, Any]", "docstring": "Extract high-level metadata from the HTML document.\n\nReturns:\n dict[str, Any]:\n Dictionary containing extracted metadata.\n\nNotes:\n **Responsibilities:**\n\n - Extract high-level metadata from the HTML document.\n - This includes: Document title, `` tag name/property to\n content mappings." } } - }, - "list": { - "name": "list", - "kind": "alias", - "path": "omniread.html.parser.list", - "signature": "", - "docstring": null - }, - "dict": { - "name": "dict", - "kind": "alias", - "path": "omniread.html.parser.dict", - "signature": "", - "docstring": null } } } diff --git a/mcp_docs/modules/omniread.html.scraper.json b/docs/mcp/modules/omniread.html.scraper.json similarity index 64% rename from mcp_docs/modules/omniread.html.scraper.json rename to docs/mcp/modules/omniread.html.scraper.json index e1865d9..2eda9e9 100644 --- a/mcp_docs/modules/omniread.html.scraper.json +++ b/docs/mcp/modules/omniread.html.scraper.json @@ -4,67 +4,39 @@ "path": "omniread.html.scraper", "docstring": "# Summary\n\nHTML scraping implementation for OmniRead.\n\nThis module provides an HTTP-based scraper for retrieving HTML documents.\nIt implements the core `BaseScraper` contract using `httpx` as the transport\nlayer.\n\nThis scraper is responsible for:\n\n- Fetching raw HTML bytes over HTTP(S)\n- Validating response content type\n- Attaching HTTP metadata to the returned content\n\nThis scraper is not responsible for:\n\n- Parsing or interpreting HTML\n- Retrying failed requests\n- Managing crawl policies or rate limiting", "objects": { - "httpx": { - "name": "httpx", - "kind": "alias", - "path": "omniread.html.scraper.httpx", - "signature": "", - "docstring": null - }, - "Any": { - "name": "Any", - "kind": "alias", - "path": "omniread.html.scraper.Any", - "signature": "", - "docstring": null - }, - "Mapping": { - "name": "Mapping", - "kind": "alias", - "path": "omniread.html.scraper.Mapping", - "signature": "", - "docstring": null - }, - "Optional": { - "name": "Optional", - "kind": "alias", - "path": "omniread.html.scraper.Optional", - "signature": "", - "docstring": null - }, "Content": { "name": "Content", "kind": "class", "path": "omniread.html.scraper.Content", - "signature": "", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.html.scraper.Content.raw", - "signature": "", + "signature": null, "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.html.scraper.Content.source", - "signature": "", + "signature": null, "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.html.scraper.Content.content_type", - "signature": "", + "signature": null, "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.html.scraper.Content.metadata", - "signature": "", + "signature": null, "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } @@ -73,35 +45,49 @@ "name": "ContentType", "kind": "class", "path": "omniread.html.scraper.ContentType", - "signature": "", + "signature": null, "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", "kind": "attribute", "path": "omniread.html.scraper.ContentType.HTML", - "signature": "", + "signature": null, "docstring": "HTML document content." }, "PDF": { "name": "PDF", "kind": "attribute", "path": "omniread.html.scraper.ContentType.PDF", - "signature": "", + "signature": null, "docstring": "PDF document content." }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.html.scraper.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.html.scraper.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, "JSON": { "name": "JSON", "kind": "attribute", "path": "omniread.html.scraper.ContentType.JSON", - "signature": "", + "signature": null, "docstring": "JSON document content." }, "XML": { "name": "XML", "kind": "attribute", "path": "omniread.html.scraper.ContentType.XML", - "signature": "", + "signature": null, "docstring": "XML document content." } } @@ -110,15 +96,15 @@ "name": "BaseScraper", "kind": "class", "path": "omniread.html.scraper.BaseScraper", - "signature": "", + "signature": null, "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.html.scraper.BaseScraper.fetch", - "signature": "", - "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." + "signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None)", + "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." } } }, @@ -126,7 +112,7 @@ "name": "HTMLScraper", "kind": "class", "path": "omniread.html.scraper.HTMLScraper", - "signature": "", + "signature": "HTMLScraper(*, client: httpx.Client | None = None, timeout: float = 15.0, headers: Mapping[str, str] | None = None, follow_redirects: bool = True)", "docstring": "Base HTML scraper using `httpx`.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns\n them as raw content wrapped in a `Content` object.\n - Fetches raw bytes and metadata only.\n - The scraper uses `httpx.Client` for HTTP requests, enforces an\n HTML content type, and preserves HTTP response metadata.\n\n **Constraints:**\n\n - The scraper does not: Parse HTML, perform retries or backoff,\n handle non-HTML responses.", "members": { "content_type": { @@ -140,15 +126,15 @@ "name": "validate_content_type", "kind": "function", "path": "omniread.html.scraper.HTMLScraper.validate_content_type", - "signature": "", + "signature": "validate_content_type(response: httpx.Response) -> None", "docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content." }, "fetch": { "name": "fetch", "kind": "function", "path": "omniread.html.scraper.HTMLScraper.fetch", - "signature": "", - "docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML." + "signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None) -> Content", + "docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML." } } } diff --git a/docs/mcp/modules/omniread.json b/docs/mcp/modules/omniread.json new file mode 100644 index 0000000..69dbd9a --- /dev/null +++ b/docs/mcp/modules/omniread.json @@ -0,0 +1,2699 @@ +{ + "module": "omniread", + "content": { + "path": "omniread", + "docstring": "# Summary\n\n`OmniRead` — format-agnostic content acquisition and parsing framework.\n\n`OmniRead` provides a **cleanly layered architecture** for fetching, parsing,\nand normalizing content from heterogeneous sources such as HTML documents\nand PDF files.\n\nThe library is structured around three core concepts:\n\n1. **`Content`**: A canonical, format-agnostic container representing raw content\n bytes and minimal contextual metadata.\n2. **`Scrapers`**: Components responsible for *acquiring* raw content from a\n source (HTTP, filesystem, object storage, etc.). `Scrapers` never interpret\n content.\n3. **`Parsers`**: Components responsible for *interpreting* acquired content and\n converting it into structured, typed representations.\n\n`OmniRead` deliberately separates these responsibilities to ensure:\n\n- Clear boundaries between IO and interpretation.\n- Replaceable implementations per format.\n- Predictable, testable behavior.\n\n# Installation\n\nInstall `OmniRead` using pip:\n\n```bash\npip install omniread\n```\n\nInstall OmniRead using Poetry:\n```bash\npoetry add omniread\n```\n\n---\n\n## Quick start\n\nExample:\n HTML example:\n ```python\n from omniread import HTMLScraper, HTMLParser\n\n scraper = HTMLScraper()\n content = scraper.fetch(\"https://example.com\")\n\n class TitleParser(HTMLParser[str]):\n def parse(self) -> str:\n return self._soup.title.string\n\n parser = TitleParser(content)\n title = parser.parse()\n ```\n\n PDF example:\n ```python\n from omniread import FileSystemPDFClient, PDFScraper, PDFParser\n from pathlib import Path\n\n client = FileSystemPDFClient()\n scraper = PDFScraper(client=client)\n content = scraper.fetch(Path(\"document.pdf\"))\n\n class TextPDFParser(PDFParser[str]):\n def parse(self) -> str:\n # implement PDF text extraction\n ...\n\n parser = TextPDFParser(content)\n result = parser.parse()\n ```\n\n---\n\n# Public API\n\nThis module re-exports the **recommended public entry points** of OmniRead.\nConsumers are encouraged to import from this namespace rather than from\nformat-specific submodules directly, unless advanced customization is\nrequired.\n\n- `Content`: Canonical content model.\n- `ContentType`: Supported media types.\n- `HTMLScraper`: HTTP-based HTML acquisition.\n- `HTMLParser`: Base parser for HTML DOM interpretation.\n- `FileSystemPDFClient`: Local filesystem PDF access.\n- `PDFScraper`: PDF-specific content acquisition.\n- `PDFParser`: Base parser for PDF binary interpretation.\n- `FileSystemXlsxClient`: Local filesystem spreadsheet access.\n- `XlsxScraper`: XLSX-specific content acquisition.\n- `XlsxParser`: Generic string-row parser for xlsx workbooks.\n\n---\n\n# Core Philosophy\n\n`OmniRead` is designed as a **decoupled content engine**:\n\n1. **Separation of Concerns**: Scrapers *fetch*, Parsers *interpret*. Neither\n knows about the other.\n2. **Normalized Exchange**: All components communicate via the `Content` model,\n ensuring a consistent contract.\n3. **Format Agnosticism**: The core logic is independent of whether the input\n is HTML, PDF, or JSON.\n\n---", + "objects": { + "Content": { + "name": "Content", + "kind": "class", + "path": "omniread.Content", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", + "members": { + "raw": { + "name": "raw", + "kind": "attribute", + "path": "omniread.Content.raw", + "signature": null, + "docstring": "Raw content bytes as retrieved from the source." + }, + "source": { + "name": "source", + "kind": "attribute", + "path": "omniread.Content.source", + "signature": null, + "docstring": "Identifier of the content origin (URL, file path, or logical name)." + }, + "content_type": { + "name": "content_type", + "kind": "attribute", + "path": "omniread.Content.content_type", + "signature": null, + "docstring": "Optional MIME type of the content, if known." + }, + "metadata": { + "name": "metadata", + "kind": "attribute", + "path": "omniread.Content.metadata", + "signature": null, + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." + } + } + }, + "ContentType": { + "name": "ContentType", + "kind": "class", + "path": "omniread.ContentType", + "signature": null, + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", + "members": { + "HTML": { + "name": "HTML", + "kind": "attribute", + "path": "omniread.ContentType.HTML", + "signature": null, + "docstring": "HTML document content." + }, + "PDF": { + "name": "PDF", + "kind": "attribute", + "path": "omniread.ContentType.PDF", + "signature": null, + "docstring": "PDF document content." + }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, + "JSON": { + "name": "JSON", + "kind": "attribute", + "path": "omniread.ContentType.JSON", + "signature": null, + "docstring": "JSON document content." + }, + "XML": { + "name": "XML", + "kind": "attribute", + "path": "omniread.ContentType.XML", + "signature": null, + "docstring": "XML document content." + } + } + }, + "BaseCsvClient": { + "name": "BaseCsvClient", + "kind": "class", + "path": "omniread.BaseCsvClient", + "signature": null, + "docstring": "Abstract client responsible for retrieving csv bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full csv binary payload.\n - Raise retrieval-specific errors on failure.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.BaseCsvClient.fetch", + "signature": "fetch(source: Any)", + "docstring": "Fetch raw csv bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." + } + } + }, + "CsvParser": { + "name": "CsvParser", + "kind": "class", + "path": "omniread.CsvParser", + "signature": "CsvParser(content: Content)", + "docstring": "Generic csv parser producing string rows from the document.\n\nNotes:\n **Responsibilities:**\n\n - Decode the payload (UTF-8 with BOM support, Latin-1 fallback).\n - Detect the delimiter from a leading sample (`,` `;` tab `|`),\n defaulting to `,`.\n - Normalize cells into deterministic stripped string values.\n - Expose row extraction helpers mirroring `XlsxParser.rows`.\n\n **Constraints:**\n\n - All values are strings; consumers requiring typed values must\n convert on their side.\n - Quoted fields containing delimiters/newlines are handled by\n the standard ``csv`` module.", + "members": { + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.CsvParser.parse", + "signature": "parse()", + "docstring": "Parse the document into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the document." + }, + "rows": { + "name": "rows", + "kind": "function", + "path": "omniread.CsvParser.rows", + "signature": "rows(*, skip_empty: bool = True)", + "docstring": "Extract normalized string rows from the document.\n\nArgs:\n skip_empty (bool):\n When True (default), rows whose cells are all blank are\n omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row." + } + } + }, + "CsvParserBase": { + "name": "CsvParserBase", + "kind": "class", + "path": "omniread.CsvParserBase", + "signature": null, + "docstring": "Base csv parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces csv content-type compatibility and provides\n the extension point for implementing concrete csv parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.CsvParserBase.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser (CSV only)." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.CsvParserBase.parse", + "signature": "parse()", + "docstring": "Parse csv content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation." + } + } + }, + "CsvScraper": { + "name": "CsvScraper", + "kind": "class", + "path": "omniread.CsvScraper", + "signature": "CsvScraper(*, client: BaseCsvClient)", + "docstring": "Scraper for csv documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw csv bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n CSV content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.CsvScraper.fetch", + "signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None)", + "docstring": "Fetch a csv document from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw csv bytes, source\n identifier, CSV content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client." + } + } + }, + "FileSystemCsvClient": { + "name": "FileSystemCsvClient", + "kind": "class", + "path": "omniread.FileSystemCsvClient", + "signature": null, + "docstring": "CSV client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads csv files directly from the disk and\n returns their raw binary contents.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.FileSystemCsvClient.fetch", + "signature": "fetch(path: Path)", + "docstring": "Read a csv file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the csv file.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." + } + } + }, + "HTMLScraper": { + "name": "HTMLScraper", + "kind": "class", + "path": "omniread.HTMLScraper", + "signature": "HTMLScraper(*, client: httpx.Client | None = None, timeout: float = 15.0, headers: Mapping[str, str] | None = None, follow_redirects: bool = True)", + "docstring": "Base HTML scraper using `httpx`.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns\n them as raw content wrapped in a `Content` object.\n - Fetches raw bytes and metadata only.\n - The scraper uses `httpx.Client` for HTTP requests, enforces an\n HTML content type, and preserves HTTP response metadata.\n\n **Constraints:**\n\n - The scraper does not: Parse HTML, perform retries or backoff,\n handle non-HTML responses.", + "members": { + "content_type": { + "name": "content_type", + "kind": "attribute", + "path": "omniread.HTMLScraper.content_type", + "signature": null, + "docstring": null + }, + "validate_content_type": { + "name": "validate_content_type", + "kind": "function", + "path": "omniread.HTMLScraper.validate_content_type", + "signature": "validate_content_type(response: httpx.Response)", + "docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content." + }, + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.HTMLScraper.fetch", + "signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None)", + "docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML." + } + } + }, + "HTMLParser": { + "name": "HTMLParser", + "kind": "class", + "path": "omniread.HTMLParser", + "signature": "HTMLParser(content: Content, features: str = 'html.parser')", + "docstring": "Base HTML parser.\n\nNotes:\n **Responsibilities:**\n\n - This class extends the core `BaseParser` with HTML-specific behavior,\n including DOM parsing via BeautifulSoup and reusable extraction helpers.\n - Provides reusable helpers for HTML extraction. Concrete parsers must\n explicitly define the return type.\n\n **Guarantees:**\n\n - Accepts only HTML content.\n - Owns a parsed BeautifulSoup DOM tree.\n - Provides pure helper utilities for common HTML structures.\n\n **Constraints:**\n\n - Concrete subclasses must define the output type `T` and implement\n the `parse()` method.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.HTMLParser.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser (HTML only)." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.HTMLParser.parse", + "signature": "parse()", + "docstring": "Fully parse the HTML content into structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the HTML DOM and return a\n deterministic, structured output." + }, + "parse_div": { + "name": "parse_div", + "kind": "function", + "path": "omniread.HTMLParser.parse_div", + "signature": "parse_div(div: Tag, *, separator: str = ' ')", + "docstring": "Extract normalized text from a `
` element.\n\nArgs:\n div (Tag):\n BeautifulSoup tag representing a `
`.\n\nReturns:\n list[list[str]]:\n A list of rows, where each row is a list of cell text values." + }, + "parse_meta": { + "name": "parse_meta", + "kind": "function", + "path": "omniread.HTMLParser.parse_meta", + "signature": "parse_meta()", + "docstring": "Extract high-level metadata from the HTML document.\n\nReturns:\n dict[str, Any]:\n Dictionary containing extracted metadata.\n\nNotes:\n **Responsibilities:**\n\n - Extract high-level metadata from the HTML document.\n - This includes: Document title, `` tag name/property to\n content mappings." + } + } + }, + "FileSystemPDFClient": { + "name": "FileSystemPDFClient", + "kind": "class", + "path": "omniread.FileSystemPDFClient", + "signature": null, + "docstring": "PDF client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads PDF files directly from the disk and returns\n their raw binary contents.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.FileSystemPDFClient.fetch", + "signature": "fetch(path: Path)", + "docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the PDF file.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." + } + } + }, + "PDFScraper": { + "name": "PDFScraper", + "kind": "class", + "path": "omniread.PDFScraper", + "signature": "PDFScraper(*, client: BasePDFClient)", + "docstring": "Scraper for PDF sources.\n\nNotes:\n **Responsibilities:**\n\n - Delegates byte retrieval to a PDF client and normalizes output\n into `Content`.\n - Preserves caller-provided metadata.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.PDFScraper.fetch", + "signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None)", + "docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client." + } + } + }, + "PDFParser": { + "name": "PDFParser", + "kind": "class", + "path": "omniread.PDFParser", + "signature": null, + "docstring": "Base PDF parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces PDF content-type compatibility and provides\n the extension point for implementing concrete PDF parsing strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.PDFParser.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser (PDF only)." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.PDFParser.parse", + "signature": "parse()", + "docstring": "Parse PDF content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the PDF binary payload and\n return a deterministic, structured output." + } + } + }, + "BaseXlsxClient": { + "name": "BaseXlsxClient", + "kind": "class", + "path": "omniread.BaseXlsxClient", + "signature": null, + "docstring": "Abstract client responsible for retrieving spreadsheet bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full xlsx binary payload.\n - Raise retrieval-specific errors on failure.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.BaseXlsxClient.fetch", + "signature": "fetch(source: Any)", + "docstring": "Fetch raw xlsx bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." + } + } + }, + "FileSystemXlsxClient": { + "name": "FileSystemXlsxClient", + "kind": "class", + "path": "omniread.FileSystemXlsxClient", + "signature": null, + "docstring": "XLSX client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads spreadsheet files directly from the disk and\n returns their raw binary contents.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.FileSystemXlsxClient.fetch", + "signature": "fetch(path: Path)", + "docstring": "Read an xlsx file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the spreadsheet file.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." + } + } + }, + "XlsxParser": { + "name": "XlsxParser", + "kind": "class", + "path": "omniread.XlsxParser", + "signature": "XlsxParser(content: Content, *, data_only: bool = True, read_only: bool = True)", + "docstring": "Generic xlsx parser producing string rows from a worksheet.\n\nNotes:\n **Responsibilities:**\n\n - Lazily load the workbook owned by the parser's content.\n - Normalize cells (including dates and numeric values) into\n deterministic string representations.\n - Expose sheet discovery and row extraction helpers.\n\n **Constraints:**\n\n - Cells are rendered with ``str(value)`` after trimming; date and\n datetime values are rendered in ISO format. Consumers requiring\n locale-specific formatting must convert on their side.", + "members": { + "workbook": { + "name": "workbook", + "kind": "attribute", + "path": "omniread.XlsxParser.workbook", + "signature": null, + "docstring": "The lazily loaded workbook backing this parser's content." + }, + "sheet_names": { + "name": "sheet_names", + "kind": "attribute", + "path": "omniread.XlsxParser.sheet_names", + "signature": null, + "docstring": "Names of all worksheets contained in the workbook." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.XlsxParser.parse", + "signature": "parse()", + "docstring": "Parse the first worksheet into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the default (first) worksheet." + }, + "rows": { + "name": "rows", + "kind": "function", + "path": "omniread.XlsxParser.rows", + "signature": "rows(sheet: int | str | None = None, *, skip_empty: bool = True)", + "docstring": "Extract normalized string rows from a worksheet.\n\nArgs:\n sheet (int | str | None):\n Worksheet index or title; defaults to the first worksheet.\n skip_empty (bool):\n When True (default), rows whose cells are all blank are omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row.\n\nRaises:\n ValueError:\n If the requested sheet does not exist." + } + } + }, + "XlsxParserBase": { + "name": "XlsxParserBase", + "kind": "class", + "path": "omniread.XlsxParserBase", + "signature": null, + "docstring": "Base xlsx parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces xlsx content-type compatibility and provides\n the extension point for implementing concrete xlsx parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.XlsxParserBase.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser (XLSX only)." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.XlsxParserBase.parse", + "signature": "parse()", + "docstring": "Parse xlsx content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation." + } + } + }, + "XlsxScraper": { + "name": "XlsxScraper", + "kind": "class", + "path": "omniread.XlsxScraper", + "signature": "XlsxScraper(*, client: BaseXlsxClient)", + "docstring": "Scraper for xlsx spreadsheet documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw xlsx bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n XLSX content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.XlsxScraper.fetch", + "signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None)", + "docstring": "Fetch an xlsx document from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw xlsx bytes, source\n identifier, XLSX content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client." + } + } + }, + "core": { + "name": "core", + "kind": "module", + "path": "omniread.core", + "signature": null, + "docstring": "# Summary\n\nCore domain contracts for OmniRead.\n\nThis package defines the **format-agnostic domain layer** of OmniRead.\nIt exposes canonical content models and abstract interfaces that are\nimplemented by format-specific modules (HTML, PDF, etc.).\n\nPublic exports from this package are considered **stable contracts** and\nare safe for downstream consumers to depend on.\n\nSubmodules:\n\n- `content`: Canonical content models and enums.\n- `parser`: Abstract parsing contracts.\n- `scraper`: Abstract scraping contracts.\n\nFormat-specific behavior must not be introduced at this layer.\n\n---\n\n# Public API\n\n- `Content`\n- `ContentType`\n\n---", + "members": { + "Content": { + "name": "Content", + "kind": "class", + "path": "omniread.core.Content", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", + "members": { + "raw": { + "name": "raw", + "kind": "attribute", + "path": "omniread.core.Content.raw", + "signature": null, + "docstring": "Raw content bytes as retrieved from the source." + }, + "source": { + "name": "source", + "kind": "attribute", + "path": "omniread.core.Content.source", + "signature": null, + "docstring": "Identifier of the content origin (URL, file path, or logical name)." + }, + "content_type": { + "name": "content_type", + "kind": "attribute", + "path": "omniread.core.Content.content_type", + "signature": null, + "docstring": "Optional MIME type of the content, if known." + }, + "metadata": { + "name": "metadata", + "kind": "attribute", + "path": "omniread.core.Content.metadata", + "signature": null, + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." + } + } + }, + "ContentType": { + "name": "ContentType", + "kind": "class", + "path": "omniread.core.ContentType", + "signature": null, + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", + "members": { + "HTML": { + "name": "HTML", + "kind": "attribute", + "path": "omniread.core.ContentType.HTML", + "signature": null, + "docstring": "HTML document content." + }, + "PDF": { + "name": "PDF", + "kind": "attribute", + "path": "omniread.core.ContentType.PDF", + "signature": null, + "docstring": "PDF document content." + }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.core.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.core.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, + "JSON": { + "name": "JSON", + "kind": "attribute", + "path": "omniread.core.ContentType.JSON", + "signature": null, + "docstring": "JSON document content." + }, + "XML": { + "name": "XML", + "kind": "attribute", + "path": "omniread.core.ContentType.XML", + "signature": null, + "docstring": "XML document content." + } + } + }, + "BaseParser": { + "name": "BaseParser", + "kind": "class", + "path": "omniread.core.BaseParser", + "signature": "BaseParser(content: Content)", + "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.core.BaseParser.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." + }, + "content": { + "name": "content", + "kind": "attribute", + "path": "omniread.core.BaseParser.content", + "signature": null, + "docstring": null + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.core.BaseParser.parse", + "signature": "parse()", + "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." + }, + "supports": { + "name": "supports", + "kind": "function", + "path": "omniread.core.BaseParser.supports", + "signature": "supports()", + "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." + } + } + }, + "BaseScraper": { + "name": "BaseScraper", + "kind": "class", + "path": "omniread.core.BaseScraper", + "signature": null, + "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.core.BaseScraper.fetch", + "signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None)", + "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." + } + } + }, + "content": { + "name": "content", + "kind": "module", + "path": "omniread.core.content", + "signature": null, + "docstring": "# Summary\n\nCanonical content models for OmniRead.\n\nThis module defines the **format-agnostic content representation** used across\nall parsers and scrapers in OmniRead.\n\nThe models defined here represent *what* was extracted, not *how* it was\nretrieved or parsed. Format-specific behavior and metadata must not alter\nthe semantic meaning of these models.", + "members": { + "ContentType": { + "name": "ContentType", + "kind": "class", + "path": "omniread.core.content.ContentType", + "signature": null, + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", + "members": { + "HTML": { + "name": "HTML", + "kind": "attribute", + "path": "omniread.core.content.ContentType.HTML", + "signature": null, + "docstring": "HTML document content." + }, + "PDF": { + "name": "PDF", + "kind": "attribute", + "path": "omniread.core.content.ContentType.PDF", + "signature": null, + "docstring": "PDF document content." + }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.core.content.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.core.content.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, + "JSON": { + "name": "JSON", + "kind": "attribute", + "path": "omniread.core.content.ContentType.JSON", + "signature": null, + "docstring": "JSON document content." + }, + "XML": { + "name": "XML", + "kind": "attribute", + "path": "omniread.core.content.ContentType.XML", + "signature": null, + "docstring": "XML document content." + } + } + }, + "Content": { + "name": "Content", + "kind": "class", + "path": "omniread.core.content.Content", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", + "members": { + "raw": { + "name": "raw", + "kind": "attribute", + "path": "omniread.core.content.Content.raw", + "signature": null, + "docstring": "Raw content bytes as retrieved from the source." + }, + "source": { + "name": "source", + "kind": "attribute", + "path": "omniread.core.content.Content.source", + "signature": null, + "docstring": "Identifier of the content origin (URL, file path, or logical name)." + }, + "content_type": { + "name": "content_type", + "kind": "attribute", + "path": "omniread.core.content.Content.content_type", + "signature": null, + "docstring": "Optional MIME type of the content, if known." + }, + "metadata": { + "name": "metadata", + "kind": "attribute", + "path": "omniread.core.content.Content.metadata", + "signature": null, + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." + } + } + } + } + }, + "parser": { + "name": "parser", + "kind": "module", + "path": "omniread.core.parser", + "signature": null, + "docstring": "# Summary\n\nAbstract parsing contracts for OmniRead.\n\nThis module defines the **format-agnostic parser interface** used to transform\nraw content into structured, typed representations.\n\nParsers are responsible for:\n\n- Interpreting a single `Content` instance\n- Validating compatibility with the content type\n- Producing a structured output suitable for downstream consumers\n\nParsers are not responsible for:\n\n- Fetching or acquiring content\n- Performing retries or error recovery\n- Managing multiple content sources", + "members": { + "Content": { + "name": "Content", + "kind": "class", + "path": "omniread.core.parser.Content", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", + "members": { + "raw": { + "name": "raw", + "kind": "attribute", + "path": "omniread.core.parser.Content.raw", + "signature": null, + "docstring": "Raw content bytes as retrieved from the source." + }, + "source": { + "name": "source", + "kind": "attribute", + "path": "omniread.core.parser.Content.source", + "signature": null, + "docstring": "Identifier of the content origin (URL, file path, or logical name)." + }, + "content_type": { + "name": "content_type", + "kind": "attribute", + "path": "omniread.core.parser.Content.content_type", + "signature": null, + "docstring": "Optional MIME type of the content, if known." + }, + "metadata": { + "name": "metadata", + "kind": "attribute", + "path": "omniread.core.parser.Content.metadata", + "signature": null, + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." + } + } + }, + "ContentType": { + "name": "ContentType", + "kind": "class", + "path": "omniread.core.parser.ContentType", + "signature": null, + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", + "members": { + "HTML": { + "name": "HTML", + "kind": "attribute", + "path": "omniread.core.parser.ContentType.HTML", + "signature": null, + "docstring": "HTML document content." + }, + "PDF": { + "name": "PDF", + "kind": "attribute", + "path": "omniread.core.parser.ContentType.PDF", + "signature": null, + "docstring": "PDF document content." + }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.core.parser.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.core.parser.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, + "JSON": { + "name": "JSON", + "kind": "attribute", + "path": "omniread.core.parser.ContentType.JSON", + "signature": null, + "docstring": "JSON document content." + }, + "XML": { + "name": "XML", + "kind": "attribute", + "path": "omniread.core.parser.ContentType.XML", + "signature": null, + "docstring": "XML document content." + } + } + }, + "T": { + "name": "T", + "kind": "attribute", + "path": "omniread.core.parser.T", + "signature": null, + "docstring": null + }, + "BaseParser": { + "name": "BaseParser", + "kind": "class", + "path": "omniread.core.parser.BaseParser", + "signature": "BaseParser(content: Content)", + "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.core.parser.BaseParser.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." + }, + "content": { + "name": "content", + "kind": "attribute", + "path": "omniread.core.parser.BaseParser.content", + "signature": null, + "docstring": null + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.core.parser.BaseParser.parse", + "signature": "parse() -> T", + "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." + }, + "supports": { + "name": "supports", + "kind": "function", + "path": "omniread.core.parser.BaseParser.supports", + "signature": "supports() -> bool", + "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." + } + } + } + } + }, + "scraper": { + "name": "scraper", + "kind": "module", + "path": "omniread.core.scraper", + "signature": null, + "docstring": "# Summary\n\nAbstract scraping contracts for OmniRead.\n\nThis module defines the **format-agnostic scraper interface** responsible for\nacquiring raw content from external sources.\n\nScrapers are responsible for:\n\n- Locating and retrieving raw content bytes\n- Attaching minimal contextual metadata\n- Returning normalized `Content` objects\n\nScrapers are explicitly NOT responsible for:\n\n- Parsing or interpreting content\n- Inferring structure or semantics\n- Performing content-type specific processing\n\nAll interpretation must be delegated to parsers.", + "members": { + "Content": { + "name": "Content", + "kind": "class", + "path": "omniread.core.scraper.Content", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", + "members": { + "raw": { + "name": "raw", + "kind": "attribute", + "path": "omniread.core.scraper.Content.raw", + "signature": null, + "docstring": "Raw content bytes as retrieved from the source." + }, + "source": { + "name": "source", + "kind": "attribute", + "path": "omniread.core.scraper.Content.source", + "signature": null, + "docstring": "Identifier of the content origin (URL, file path, or logical name)." + }, + "content_type": { + "name": "content_type", + "kind": "attribute", + "path": "omniread.core.scraper.Content.content_type", + "signature": null, + "docstring": "Optional MIME type of the content, if known." + }, + "metadata": { + "name": "metadata", + "kind": "attribute", + "path": "omniread.core.scraper.Content.metadata", + "signature": null, + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." + } + } + }, + "BaseScraper": { + "name": "BaseScraper", + "kind": "class", + "path": "omniread.core.scraper.BaseScraper", + "signature": null, + "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.core.scraper.BaseScraper.fetch", + "signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None) -> Content", + "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." + } + } + } + } + } + } + }, + "csv": { + "name": "csv", + "kind": "module", + "path": "omniread.csv", + "signature": null, + "docstring": "# Summary\n\nCSV subpackage for OmniRead.\n\nProvides acquisition and parsing of comma-separated-value content:\n\n- `BaseCsvClient`: abstract backing-store client for csv bytes.\n- `FileSystemCsvClient`: local filesystem implementation.\n- `CsvScraper`: wraps fetched bytes into canonical `Content`.\n- `CsvParserBase`: content-type-enforcing parser contract.\n- `CsvParser`: generic string-row parser built on the standard csv module.", + "members": { + "BaseCsvClient": { + "name": "BaseCsvClient", + "kind": "class", + "path": "omniread.csv.BaseCsvClient", + "signature": null, + "docstring": "Abstract client responsible for retrieving csv bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full csv binary payload.\n - Raise retrieval-specific errors on failure.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.csv.BaseCsvClient.fetch", + "signature": "fetch(source: Any)", + "docstring": "Fetch raw csv bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." + } + } + }, + "FileSystemCsvClient": { + "name": "FileSystemCsvClient", + "kind": "class", + "path": "omniread.csv.FileSystemCsvClient", + "signature": null, + "docstring": "CSV client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads csv files directly from the disk and\n returns their raw binary contents.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.csv.FileSystemCsvClient.fetch", + "signature": "fetch(path: Path)", + "docstring": "Read a csv file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the csv file.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." + } + } + }, + "CsvParser": { + "name": "CsvParser", + "kind": "class", + "path": "omniread.csv.CsvParser", + "signature": "CsvParser(content: Content)", + "docstring": "Generic csv parser producing string rows from the document.\n\nNotes:\n **Responsibilities:**\n\n - Decode the payload (UTF-8 with BOM support, Latin-1 fallback).\n - Detect the delimiter from a leading sample (`,` `;` tab `|`),\n defaulting to `,`.\n - Normalize cells into deterministic stripped string values.\n - Expose row extraction helpers mirroring `XlsxParser.rows`.\n\n **Constraints:**\n\n - All values are strings; consumers requiring typed values must\n convert on their side.\n - Quoted fields containing delimiters/newlines are handled by\n the standard ``csv`` module.", + "members": { + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.csv.CsvParser.parse", + "signature": "parse()", + "docstring": "Parse the document into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the document." + }, + "rows": { + "name": "rows", + "kind": "function", + "path": "omniread.csv.CsvParser.rows", + "signature": "rows(*, skip_empty: bool = True)", + "docstring": "Extract normalized string rows from the document.\n\nArgs:\n skip_empty (bool):\n When True (default), rows whose cells are all blank are\n omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row." + } + } + }, + "CsvParserBase": { + "name": "CsvParserBase", + "kind": "class", + "path": "omniread.csv.CsvParserBase", + "signature": null, + "docstring": "Base csv parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces csv content-type compatibility and provides\n the extension point for implementing concrete csv parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.csv.CsvParserBase.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser (CSV only)." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.csv.CsvParserBase.parse", + "signature": "parse()", + "docstring": "Parse csv content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation." + } + } + }, + "CsvScraper": { + "name": "CsvScraper", + "kind": "class", + "path": "omniread.csv.CsvScraper", + "signature": "CsvScraper(*, client: BaseCsvClient)", + "docstring": "Scraper for csv documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw csv bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n CSV content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.csv.CsvScraper.fetch", + "signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None)", + "docstring": "Fetch a csv document from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw csv bytes, source\n identifier, CSV content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client." + } + } + }, + "client": { + "name": "client", + "kind": "module", + "path": "omniread.csv.client", + "signature": null, + "docstring": "# Summary\n\nCSV client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw\ncomma-separated-value document bytes from a concrete backing store.\n\nClients provide low-level access to csv binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems", + "members": { + "BaseCsvClient": { + "name": "BaseCsvClient", + "kind": "class", + "path": "omniread.csv.client.BaseCsvClient", + "signature": null, + "docstring": "Abstract client responsible for retrieving csv bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full csv binary payload.\n - Raise retrieval-specific errors on failure.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.csv.client.BaseCsvClient.fetch", + "signature": "fetch(source: Any) -> bytes", + "docstring": "Fetch raw csv bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." + } + } + }, + "FileSystemCsvClient": { + "name": "FileSystemCsvClient", + "kind": "class", + "path": "omniread.csv.client.FileSystemCsvClient", + "signature": null, + "docstring": "CSV client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads csv files directly from the disk and\n returns their raw binary contents.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.csv.client.FileSystemCsvClient.fetch", + "signature": "fetch(path: Path) -> bytes", + "docstring": "Read a csv file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the csv file.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." + } + } + } + } + }, + "parser": { + "name": "parser", + "kind": "module", + "path": "omniread.csv.parser", + "signature": null, + "docstring": "# Summary\n\nCSV parser implementations for OmniRead.\n\nThis module provides a concrete, generic parser for comma-separated-value\ndocuments. It exposes records as lists of string cells so downstream\nconsumers can interpret tabular content without depending on the ``csv``\nmodule directly.\n\nThe parser is intentionally statement-agnostic: it performs no header\ndetection or column interpretation beyond basic cell normalization and\ndelimiter detection.", + "members": { + "Content": { + "name": "Content", + "kind": "class", + "path": "omniread.csv.parser.Content", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", + "members": { + "raw": { + "name": "raw", + "kind": "attribute", + "path": "omniread.csv.parser.Content.raw", + "signature": null, + "docstring": "Raw content bytes as retrieved from the source." + }, + "source": { + "name": "source", + "kind": "attribute", + "path": "omniread.csv.parser.Content.source", + "signature": null, + "docstring": "Identifier of the content origin (URL, file path, or logical name)." + }, + "content_type": { + "name": "content_type", + "kind": "attribute", + "path": "omniread.csv.parser.Content.content_type", + "signature": null, + "docstring": "Optional MIME type of the content, if known." + }, + "metadata": { + "name": "metadata", + "kind": "attribute", + "path": "omniread.csv.parser.Content.metadata", + "signature": null, + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." + } + } + }, + "CsvParserBase": { + "name": "CsvParserBase", + "kind": "class", + "path": "omniread.csv.parser.CsvParserBase", + "signature": null, + "docstring": "Base csv parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces csv content-type compatibility and provides\n the extension point for implementing concrete csv parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.csv.parser.CsvParserBase.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser (CSV only)." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.csv.parser.CsvParserBase.parse", + "signature": "parse()", + "docstring": "Parse csv content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation." + } + } + }, + "CsvParser": { + "name": "CsvParser", + "kind": "class", + "path": "omniread.csv.parser.CsvParser", + "signature": "CsvParser(content: Content)", + "docstring": "Generic csv parser producing string rows from the document.\n\nNotes:\n **Responsibilities:**\n\n - Decode the payload (UTF-8 with BOM support, Latin-1 fallback).\n - Detect the delimiter from a leading sample (`,` `;` tab `|`),\n defaulting to `,`.\n - Normalize cells into deterministic stripped string values.\n - Expose row extraction helpers mirroring `XlsxParser.rows`.\n\n **Constraints:**\n\n - All values are strings; consumers requiring typed values must\n convert on their side.\n - Quoted fields containing delimiters/newlines are handled by\n the standard ``csv`` module.", + "members": { + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.csv.parser.CsvParser.parse", + "signature": "parse() -> list[list[str]]", + "docstring": "Parse the document into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the document." + }, + "rows": { + "name": "rows", + "kind": "function", + "path": "omniread.csv.parser.CsvParser.rows", + "signature": "rows(*, skip_empty: bool = True) -> list[list[str]]", + "docstring": "Extract normalized string rows from the document.\n\nArgs:\n skip_empty (bool):\n When True (default), rows whose cells are all blank are\n omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row." + } + } + } + } + }, + "parser_base": { + "name": "parser_base", + "kind": "module", + "path": "omniread.csv.parser_base", + "signature": null, + "docstring": "# Summary\n\nCSV parser base implementation for OmniRead.\n\nThis module defines the **CSV-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for\ncomma-separated-value documents.", + "members": { + "ContentType": { + "name": "ContentType", + "kind": "class", + "path": "omniread.csv.parser_base.ContentType", + "signature": null, + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", + "members": { + "HTML": { + "name": "HTML", + "kind": "attribute", + "path": "omniread.csv.parser_base.ContentType.HTML", + "signature": null, + "docstring": "HTML document content." + }, + "PDF": { + "name": "PDF", + "kind": "attribute", + "path": "omniread.csv.parser_base.ContentType.PDF", + "signature": null, + "docstring": "PDF document content." + }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.csv.parser_base.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.csv.parser_base.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, + "JSON": { + "name": "JSON", + "kind": "attribute", + "path": "omniread.csv.parser_base.ContentType.JSON", + "signature": null, + "docstring": "JSON document content." + }, + "XML": { + "name": "XML", + "kind": "attribute", + "path": "omniread.csv.parser_base.ContentType.XML", + "signature": null, + "docstring": "XML document content." + } + } + }, + "BaseParser": { + "name": "BaseParser", + "kind": "class", + "path": "omniread.csv.parser_base.BaseParser", + "signature": "BaseParser(content: Content)", + "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.csv.parser_base.BaseParser.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." + }, + "content": { + "name": "content", + "kind": "attribute", + "path": "omniread.csv.parser_base.BaseParser.content", + "signature": null, + "docstring": null + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.csv.parser_base.BaseParser.parse", + "signature": "parse()", + "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." + }, + "supports": { + "name": "supports", + "kind": "function", + "path": "omniread.csv.parser_base.BaseParser.supports", + "signature": "supports()", + "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." + } + } + }, + "T": { + "name": "T", + "kind": "attribute", + "path": "omniread.csv.parser_base.T", + "signature": null, + "docstring": null + }, + "CsvParserBase": { + "name": "CsvParserBase", + "kind": "class", + "path": "omniread.csv.parser_base.CsvParserBase", + "signature": null, + "docstring": "Base csv parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces csv content-type compatibility and provides\n the extension point for implementing concrete csv parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.csv.parser_base.CsvParserBase.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser (CSV only)." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.csv.parser_base.CsvParserBase.parse", + "signature": "parse() -> T", + "docstring": "Parse csv content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation." + } + } + } + } + }, + "scraper": { + "name": "scraper", + "kind": "module", + "path": "omniread.csv.scraper", + "signature": null, + "docstring": "# Summary\n\nCSV scraper for OmniRead.\n\nThis module defines the scraper responsible for acquiring raw\ncomma-separated-value document content from a backing store via a\nconfigured client.\n\nThe scraper does not interpret or parse the acquired bytes; it wraps them in\nthe canonical `Content` model.", + "members": { + "Content": { + "name": "Content", + "kind": "class", + "path": "omniread.csv.scraper.Content", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", + "members": { + "raw": { + "name": "raw", + "kind": "attribute", + "path": "omniread.csv.scraper.Content.raw", + "signature": null, + "docstring": "Raw content bytes as retrieved from the source." + }, + "source": { + "name": "source", + "kind": "attribute", + "path": "omniread.csv.scraper.Content.source", + "signature": null, + "docstring": "Identifier of the content origin (URL, file path, or logical name)." + }, + "content_type": { + "name": "content_type", + "kind": "attribute", + "path": "omniread.csv.scraper.Content.content_type", + "signature": null, + "docstring": "Optional MIME type of the content, if known." + }, + "metadata": { + "name": "metadata", + "kind": "attribute", + "path": "omniread.csv.scraper.Content.metadata", + "signature": null, + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." + } + } + }, + "ContentType": { + "name": "ContentType", + "kind": "class", + "path": "omniread.csv.scraper.ContentType", + "signature": null, + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", + "members": { + "HTML": { + "name": "HTML", + "kind": "attribute", + "path": "omniread.csv.scraper.ContentType.HTML", + "signature": null, + "docstring": "HTML document content." + }, + "PDF": { + "name": "PDF", + "kind": "attribute", + "path": "omniread.csv.scraper.ContentType.PDF", + "signature": null, + "docstring": "PDF document content." + }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.csv.scraper.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.csv.scraper.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, + "JSON": { + "name": "JSON", + "kind": "attribute", + "path": "omniread.csv.scraper.ContentType.JSON", + "signature": null, + "docstring": "JSON document content." + }, + "XML": { + "name": "XML", + "kind": "attribute", + "path": "omniread.csv.scraper.ContentType.XML", + "signature": null, + "docstring": "XML document content." + } + } + }, + "BaseCsvClient": { + "name": "BaseCsvClient", + "kind": "class", + "path": "omniread.csv.scraper.BaseCsvClient", + "signature": null, + "docstring": "Abstract client responsible for retrieving csv bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full csv binary payload.\n - Raise retrieval-specific errors on failure.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.csv.scraper.BaseCsvClient.fetch", + "signature": "fetch(source: Any)", + "docstring": "Fetch raw csv bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw csv bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." + } + } + }, + "CsvScraper": { + "name": "CsvScraper", + "kind": "class", + "path": "omniread.csv.scraper.CsvScraper", + "signature": "CsvScraper(*, client: BaseCsvClient)", + "docstring": "Scraper for csv documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw csv bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n CSV content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.csv.scraper.CsvScraper.fetch", + "signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None) -> Content", + "docstring": "Fetch a csv document from the given source.\n\nArgs:\n source (Any):\n Identifier of the csv source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw csv bytes, source\n identifier, CSV content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client." + } + } + } + } + } + } + }, + "html": { + "name": "html", + "kind": "module", + "path": "omniread.html", + "signature": null, + "docstring": "# Summary\n\nHTML format implementation for OmniRead.\n\nThis package provides **HTML-specific implementations** of the core OmniRead\ncontracts defined in `omniread.core`.\n\nIt includes:\n\n- HTML parsers that interpret HTML content.\n- HTML scrapers that retrieve HTML documents.\n\nKey characteristics:\n\n- Implements, but does not redefine, core contracts.\n- May contain HTML-specific behavior and edge-case handling.\n- Produces canonical content models defined in `omniread.core.content`.\n\nConsumers should depend on `omniread.core` interfaces wherever possible and\nuse this package only when HTML-specific behavior is required.\n\n---\n\n# Public API\n\n- `HTMLScraper`\n- `HTMLParser`\n\n---", + "members": { + "HTMLScraper": { + "name": "HTMLScraper", + "kind": "class", + "path": "omniread.html.HTMLScraper", + "signature": "HTMLScraper(*, client: httpx.Client | None = None, timeout: float = 15.0, headers: Mapping[str, str] | None = None, follow_redirects: bool = True)", + "docstring": "Base HTML scraper using `httpx`.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns\n them as raw content wrapped in a `Content` object.\n - Fetches raw bytes and metadata only.\n - The scraper uses `httpx.Client` for HTTP requests, enforces an\n HTML content type, and preserves HTTP response metadata.\n\n **Constraints:**\n\n - The scraper does not: Parse HTML, perform retries or backoff,\n handle non-HTML responses.", + "members": { + "content_type": { + "name": "content_type", + "kind": "attribute", + "path": "omniread.html.HTMLScraper.content_type", + "signature": null, + "docstring": null + }, + "validate_content_type": { + "name": "validate_content_type", + "kind": "function", + "path": "omniread.html.HTMLScraper.validate_content_type", + "signature": "validate_content_type(response: httpx.Response)", + "docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content." + }, + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.html.HTMLScraper.fetch", + "signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None)", + "docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML." + } + } + }, + "HTMLParser": { + "name": "HTMLParser", + "kind": "class", + "path": "omniread.html.HTMLParser", + "signature": "HTMLParser(content: Content, features: str = 'html.parser')", + "docstring": "Base HTML parser.\n\nNotes:\n **Responsibilities:**\n\n - This class extends the core `BaseParser` with HTML-specific behavior,\n including DOM parsing via BeautifulSoup and reusable extraction helpers.\n - Provides reusable helpers for HTML extraction. Concrete parsers must\n explicitly define the return type.\n\n **Guarantees:**\n\n - Accepts only HTML content.\n - Owns a parsed BeautifulSoup DOM tree.\n - Provides pure helper utilities for common HTML structures.\n\n **Constraints:**\n\n - Concrete subclasses must define the output type `T` and implement\n the `parse()` method.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.html.HTMLParser.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser (HTML only)." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.html.HTMLParser.parse", + "signature": "parse()", + "docstring": "Fully parse the HTML content into structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the HTML DOM and return a\n deterministic, structured output." + }, + "parse_div": { + "name": "parse_div", + "kind": "function", + "path": "omniread.html.HTMLParser.parse_div", + "signature": "parse_div(div: Tag, *, separator: str = ' ')", + "docstring": "Extract normalized text from a `
` element.\n\nArgs:\n div (Tag):\n BeautifulSoup tag representing a `
`.\n\nReturns:\n list[list[str]]:\n A list of rows, where each row is a list of cell text values." + }, + "parse_meta": { + "name": "parse_meta", + "kind": "function", + "path": "omniread.html.HTMLParser.parse_meta", + "signature": "parse_meta()", + "docstring": "Extract high-level metadata from the HTML document.\n\nReturns:\n dict[str, Any]:\n Dictionary containing extracted metadata.\n\nNotes:\n **Responsibilities:**\n\n - Extract high-level metadata from the HTML document.\n - This includes: Document title, `` tag name/property to\n content mappings." + } + } + }, + "parser": { + "name": "parser", + "kind": "module", + "path": "omniread.html.parser", + "signature": null, + "docstring": "# Summary\n\nHTML parser base implementations for OmniRead.\n\nThis module provides reusable HTML parsing utilities built on top of\nthe abstract parser contracts defined in `omniread.core.parser`.\n\nIt supplies:\n\n- Content-type enforcement for HTML inputs\n- BeautifulSoup initialization and lifecycle management\n- Common helper methods for extracting structured data from HTML elements\n\nConcrete parsers must subclass `HTMLParser` and implement the `parse()` method\nto return a structured representation appropriate for their use case.", + "members": { + "Content": { + "name": "Content", + "kind": "class", + "path": "omniread.html.parser.Content", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", + "members": { + "raw": { + "name": "raw", + "kind": "attribute", + "path": "omniread.html.parser.Content.raw", + "signature": null, + "docstring": "Raw content bytes as retrieved from the source." + }, + "source": { + "name": "source", + "kind": "attribute", + "path": "omniread.html.parser.Content.source", + "signature": null, + "docstring": "Identifier of the content origin (URL, file path, or logical name)." + }, + "content_type": { + "name": "content_type", + "kind": "attribute", + "path": "omniread.html.parser.Content.content_type", + "signature": null, + "docstring": "Optional MIME type of the content, if known." + }, + "metadata": { + "name": "metadata", + "kind": "attribute", + "path": "omniread.html.parser.Content.metadata", + "signature": null, + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." + } + } + }, + "ContentType": { + "name": "ContentType", + "kind": "class", + "path": "omniread.html.parser.ContentType", + "signature": null, + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", + "members": { + "HTML": { + "name": "HTML", + "kind": "attribute", + "path": "omniread.html.parser.ContentType.HTML", + "signature": null, + "docstring": "HTML document content." + }, + "PDF": { + "name": "PDF", + "kind": "attribute", + "path": "omniread.html.parser.ContentType.PDF", + "signature": null, + "docstring": "PDF document content." + }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.html.parser.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.html.parser.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, + "JSON": { + "name": "JSON", + "kind": "attribute", + "path": "omniread.html.parser.ContentType.JSON", + "signature": null, + "docstring": "JSON document content." + }, + "XML": { + "name": "XML", + "kind": "attribute", + "path": "omniread.html.parser.ContentType.XML", + "signature": null, + "docstring": "XML document content." + } + } + }, + "BaseParser": { + "name": "BaseParser", + "kind": "class", + "path": "omniread.html.parser.BaseParser", + "signature": "BaseParser(content: Content)", + "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.html.parser.BaseParser.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." + }, + "content": { + "name": "content", + "kind": "attribute", + "path": "omniread.html.parser.BaseParser.content", + "signature": null, + "docstring": null + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.html.parser.BaseParser.parse", + "signature": "parse()", + "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." + }, + "supports": { + "name": "supports", + "kind": "function", + "path": "omniread.html.parser.BaseParser.supports", + "signature": "supports()", + "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." + } + } + }, + "T": { + "name": "T", + "kind": "attribute", + "path": "omniread.html.parser.T", + "signature": null, + "docstring": null + }, + "HTMLParser": { + "name": "HTMLParser", + "kind": "class", + "path": "omniread.html.parser.HTMLParser", + "signature": "HTMLParser(content: Content, features: str = 'html.parser')", + "docstring": "Base HTML parser.\n\nNotes:\n **Responsibilities:**\n\n - This class extends the core `BaseParser` with HTML-specific behavior,\n including DOM parsing via BeautifulSoup and reusable extraction helpers.\n - Provides reusable helpers for HTML extraction. Concrete parsers must\n explicitly define the return type.\n\n **Guarantees:**\n\n - Accepts only HTML content.\n - Owns a parsed BeautifulSoup DOM tree.\n - Provides pure helper utilities for common HTML structures.\n\n **Constraints:**\n\n - Concrete subclasses must define the output type `T` and implement\n the `parse()` method.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.html.parser.HTMLParser.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser (HTML only)." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.html.parser.HTMLParser.parse", + "signature": "parse() -> T", + "docstring": "Fully parse the HTML content into structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the HTML DOM and return a\n deterministic, structured output." + }, + "parse_div": { + "name": "parse_div", + "kind": "function", + "path": "omniread.html.parser.HTMLParser.parse_div", + "signature": "parse_div(div: Tag, *, separator: str = ' ') -> str", + "docstring": "Extract normalized text from a `
` element.\n\nArgs:\n div (Tag):\n BeautifulSoup tag representing a `
`.\n separator (str, optional):\n String used to separate text nodes.\n\nReturns:\n str:\n Flattened, whitespace-normalized text content." + }, + "parse_link": { + "name": "parse_link", + "kind": "function", + "path": "omniread.html.parser.HTMLParser.parse_link", + "signature": "parse_link(a: Tag) -> str | None", + "docstring": "Extract the hyperlink reference from an `` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n str | None:\n The value of the `href` attribute, or None if absent." + }, + "parse_table": { + "name": "parse_table", + "kind": "function", + "path": "omniread.html.parser.HTMLParser.parse_table", + "signature": "parse_table(table: Tag) -> list[list[str]]", + "docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table (Tag):\n BeautifulSoup tag representing a `
`.\n\nReturns:\n list[list[str]]:\n A list of rows, where each row is a list of cell text values." + }, + "parse_meta": { + "name": "parse_meta", + "kind": "function", + "path": "omniread.html.parser.HTMLParser.parse_meta", + "signature": "parse_meta() -> dict[str, Any]", + "docstring": "Extract high-level metadata from the HTML document.\n\nReturns:\n dict[str, Any]:\n Dictionary containing extracted metadata.\n\nNotes:\n **Responsibilities:**\n\n - Extract high-level metadata from the HTML document.\n - This includes: Document title, `` tag name/property to\n content mappings." + } + } + } + } + }, + "scraper": { + "name": "scraper", + "kind": "module", + "path": "omniread.html.scraper", + "signature": null, + "docstring": "# Summary\n\nHTML scraping implementation for OmniRead.\n\nThis module provides an HTTP-based scraper for retrieving HTML documents.\nIt implements the core `BaseScraper` contract using `httpx` as the transport\nlayer.\n\nThis scraper is responsible for:\n\n- Fetching raw HTML bytes over HTTP(S)\n- Validating response content type\n- Attaching HTTP metadata to the returned content\n\nThis scraper is not responsible for:\n\n- Parsing or interpreting HTML\n- Retrying failed requests\n- Managing crawl policies or rate limiting", + "members": { + "Content": { + "name": "Content", + "kind": "class", + "path": "omniread.html.scraper.Content", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", + "members": { + "raw": { + "name": "raw", + "kind": "attribute", + "path": "omniread.html.scraper.Content.raw", + "signature": null, + "docstring": "Raw content bytes as retrieved from the source." + }, + "source": { + "name": "source", + "kind": "attribute", + "path": "omniread.html.scraper.Content.source", + "signature": null, + "docstring": "Identifier of the content origin (URL, file path, or logical name)." + }, + "content_type": { + "name": "content_type", + "kind": "attribute", + "path": "omniread.html.scraper.Content.content_type", + "signature": null, + "docstring": "Optional MIME type of the content, if known." + }, + "metadata": { + "name": "metadata", + "kind": "attribute", + "path": "omniread.html.scraper.Content.metadata", + "signature": null, + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." + } + } + }, + "ContentType": { + "name": "ContentType", + "kind": "class", + "path": "omniread.html.scraper.ContentType", + "signature": null, + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", + "members": { + "HTML": { + "name": "HTML", + "kind": "attribute", + "path": "omniread.html.scraper.ContentType.HTML", + "signature": null, + "docstring": "HTML document content." + }, + "PDF": { + "name": "PDF", + "kind": "attribute", + "path": "omniread.html.scraper.ContentType.PDF", + "signature": null, + "docstring": "PDF document content." + }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.html.scraper.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.html.scraper.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, + "JSON": { + "name": "JSON", + "kind": "attribute", + "path": "omniread.html.scraper.ContentType.JSON", + "signature": null, + "docstring": "JSON document content." + }, + "XML": { + "name": "XML", + "kind": "attribute", + "path": "omniread.html.scraper.ContentType.XML", + "signature": null, + "docstring": "XML document content." + } + } + }, + "BaseScraper": { + "name": "BaseScraper", + "kind": "class", + "path": "omniread.html.scraper.BaseScraper", + "signature": null, + "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.html.scraper.BaseScraper.fetch", + "signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None)", + "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." + } + } + }, + "HTMLScraper": { + "name": "HTMLScraper", + "kind": "class", + "path": "omniread.html.scraper.HTMLScraper", + "signature": "HTMLScraper(*, client: httpx.Client | None = None, timeout: float = 15.0, headers: Mapping[str, str] | None = None, follow_redirects: bool = True)", + "docstring": "Base HTML scraper using `httpx`.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns\n them as raw content wrapped in a `Content` object.\n - Fetches raw bytes and metadata only.\n - The scraper uses `httpx.Client` for HTTP requests, enforces an\n HTML content type, and preserves HTTP response metadata.\n\n **Constraints:**\n\n - The scraper does not: Parse HTML, perform retries or backoff,\n handle non-HTML responses.", + "members": { + "content_type": { + "name": "content_type", + "kind": "attribute", + "path": "omniread.html.scraper.HTMLScraper.content_type", + "signature": null, + "docstring": null + }, + "validate_content_type": { + "name": "validate_content_type", + "kind": "function", + "path": "omniread.html.scraper.HTMLScraper.validate_content_type", + "signature": "validate_content_type(response: httpx.Response) -> None", + "docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content." + }, + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.html.scraper.HTMLScraper.fetch", + "signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None) -> Content", + "docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML." + } + } + } + } + } + } + }, + "pdf": { + "name": "pdf", + "kind": "module", + "path": "omniread.pdf", + "signature": null, + "docstring": "# Summary\n\nPDF format implementation for OmniRead.\n\nThis package provides **PDF-specific implementations** of the core OmniRead\ncontracts defined in `omniread.core`.\n\nUnlike HTML, PDF handling requires an explicit client layer for document\naccess. This package therefore includes:\n\n- PDF clients for acquiring raw PDF data.\n- PDF scrapers that coordinate client access.\n- PDF parsers that extract structured content from PDF binaries.\n\nPublic exports from this package represent the supported PDF pipeline\nand are safe for consumers to import directly when working with PDFs.\n\n---\n\n# Public API\n\n- `FileSystemPDFClient`\n- `PDFScraper`\n- `PDFParser`\n\n---", + "members": { + "FileSystemPDFClient": { + "name": "FileSystemPDFClient", + "kind": "class", + "path": "omniread.pdf.FileSystemPDFClient", + "signature": null, + "docstring": "PDF client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads PDF files directly from the disk and returns\n their raw binary contents.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.pdf.FileSystemPDFClient.fetch", + "signature": "fetch(path: Path)", + "docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the PDF file.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." + } + } + }, + "PDFScraper": { + "name": "PDFScraper", + "kind": "class", + "path": "omniread.pdf.PDFScraper", + "signature": "PDFScraper(*, client: BasePDFClient)", + "docstring": "Scraper for PDF sources.\n\nNotes:\n **Responsibilities:**\n\n - Delegates byte retrieval to a PDF client and normalizes output\n into `Content`.\n - Preserves caller-provided metadata.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.pdf.PDFScraper.fetch", + "signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None)", + "docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client." + } + } + }, + "PDFParser": { + "name": "PDFParser", + "kind": "class", + "path": "omniread.pdf.PDFParser", + "signature": null, + "docstring": "Base PDF parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces PDF content-type compatibility and provides\n the extension point for implementing concrete PDF parsing strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.pdf.PDFParser.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser (PDF only)." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.pdf.PDFParser.parse", + "signature": "parse()", + "docstring": "Parse PDF content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the PDF binary payload and\n return a deterministic, structured output." + } + } + }, + "client": { + "name": "client", + "kind": "module", + "path": "omniread.pdf.client", + "signature": null, + "docstring": "# Summary\n\nPDF client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw PDF\nbytes from a concrete backing store.\n\nClients provide low-level access to PDF binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems", + "members": { + "BasePDFClient": { + "name": "BasePDFClient", + "kind": "class", + "path": "omniread.pdf.client.BasePDFClient", + "signature": null, + "docstring": "Abstract client responsible for retrieving PDF bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full PDF binary payload.\n - Raise retrieval-specific errors on failure.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.pdf.client.BasePDFClient.fetch", + "signature": "fetch(source: Any) -> bytes", + "docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF location, such as a file path, object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." + } + } + }, + "FileSystemPDFClient": { + "name": "FileSystemPDFClient", + "kind": "class", + "path": "omniread.pdf.client.FileSystemPDFClient", + "signature": null, + "docstring": "PDF client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads PDF files directly from the disk and returns\n their raw binary contents.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.pdf.client.FileSystemPDFClient.fetch", + "signature": "fetch(path: Path) -> bytes", + "docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the PDF file.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." + } + } + } + } + }, + "parser": { + "name": "parser", + "kind": "module", + "path": "omniread.pdf.parser", + "signature": null, + "docstring": "# Summary\n\nPDF parser base implementations for OmniRead.\n\nThis module defines the **PDF-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for PDF content.\n\nPDF parsers are responsible for interpreting binary PDF data and producing\nstructured representations suitable for downstream consumption.", + "members": { + "ContentType": { + "name": "ContentType", + "kind": "class", + "path": "omniread.pdf.parser.ContentType", + "signature": null, + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", + "members": { + "HTML": { + "name": "HTML", + "kind": "attribute", + "path": "omniread.pdf.parser.ContentType.HTML", + "signature": null, + "docstring": "HTML document content." + }, + "PDF": { + "name": "PDF", + "kind": "attribute", + "path": "omniread.pdf.parser.ContentType.PDF", + "signature": null, + "docstring": "PDF document content." + }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.pdf.parser.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.pdf.parser.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, + "JSON": { + "name": "JSON", + "kind": "attribute", + "path": "omniread.pdf.parser.ContentType.JSON", + "signature": null, + "docstring": "JSON document content." + }, + "XML": { + "name": "XML", + "kind": "attribute", + "path": "omniread.pdf.parser.ContentType.XML", + "signature": null, + "docstring": "XML document content." + } + } + }, + "BaseParser": { + "name": "BaseParser", + "kind": "class", + "path": "omniread.pdf.parser.BaseParser", + "signature": "BaseParser(content: Content)", + "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.pdf.parser.BaseParser.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." + }, + "content": { + "name": "content", + "kind": "attribute", + "path": "omniread.pdf.parser.BaseParser.content", + "signature": null, + "docstring": null + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.pdf.parser.BaseParser.parse", + "signature": "parse()", + "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." + }, + "supports": { + "name": "supports", + "kind": "function", + "path": "omniread.pdf.parser.BaseParser.supports", + "signature": "supports()", + "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." + } + } + }, + "T": { + "name": "T", + "kind": "attribute", + "path": "omniread.pdf.parser.T", + "signature": null, + "docstring": null + }, + "PDFParser": { + "name": "PDFParser", + "kind": "class", + "path": "omniread.pdf.parser.PDFParser", + "signature": null, + "docstring": "Base PDF parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces PDF content-type compatibility and provides\n the extension point for implementing concrete PDF parsing strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.pdf.parser.PDFParser.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser (PDF only)." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.pdf.parser.PDFParser.parse", + "signature": "parse() -> T", + "docstring": "Parse PDF content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the PDF binary payload and\n return a deterministic, structured output." + } + } + } + } + }, + "scraper": { + "name": "scraper", + "kind": "module", + "path": "omniread.pdf.scraper", + "signature": null, + "docstring": "# Summary\n\nPDF scraping implementation for OmniRead.\n\nThis module provides a PDF-specific scraper that coordinates PDF byte\nretrieval via a client and normalizes the result into a `Content` object.\n\nThe scraper implements the core `BaseScraper` contract while delegating\nall storage and access concerns to a `BasePDFClient` implementation.", + "members": { + "Content": { + "name": "Content", + "kind": "class", + "path": "omniread.pdf.scraper.Content", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", + "members": { + "raw": { + "name": "raw", + "kind": "attribute", + "path": "omniread.pdf.scraper.Content.raw", + "signature": null, + "docstring": "Raw content bytes as retrieved from the source." + }, + "source": { + "name": "source", + "kind": "attribute", + "path": "omniread.pdf.scraper.Content.source", + "signature": null, + "docstring": "Identifier of the content origin (URL, file path, or logical name)." + }, + "content_type": { + "name": "content_type", + "kind": "attribute", + "path": "omniread.pdf.scraper.Content.content_type", + "signature": null, + "docstring": "Optional MIME type of the content, if known." + }, + "metadata": { + "name": "metadata", + "kind": "attribute", + "path": "omniread.pdf.scraper.Content.metadata", + "signature": null, + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." + } + } + }, + "ContentType": { + "name": "ContentType", + "kind": "class", + "path": "omniread.pdf.scraper.ContentType", + "signature": null, + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", + "members": { + "HTML": { + "name": "HTML", + "kind": "attribute", + "path": "omniread.pdf.scraper.ContentType.HTML", + "signature": null, + "docstring": "HTML document content." + }, + "PDF": { + "name": "PDF", + "kind": "attribute", + "path": "omniread.pdf.scraper.ContentType.PDF", + "signature": null, + "docstring": "PDF document content." + }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.pdf.scraper.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.pdf.scraper.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, + "JSON": { + "name": "JSON", + "kind": "attribute", + "path": "omniread.pdf.scraper.ContentType.JSON", + "signature": null, + "docstring": "JSON document content." + }, + "XML": { + "name": "XML", + "kind": "attribute", + "path": "omniread.pdf.scraper.ContentType.XML", + "signature": null, + "docstring": "XML document content." + } + } + }, + "BaseScraper": { + "name": "BaseScraper", + "kind": "class", + "path": "omniread.pdf.scraper.BaseScraper", + "signature": null, + "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.pdf.scraper.BaseScraper.fetch", + "signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None)", + "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." + } + } + }, + "BasePDFClient": { + "name": "BasePDFClient", + "kind": "class", + "path": "omniread.pdf.scraper.BasePDFClient", + "signature": null, + "docstring": "Abstract client responsible for retrieving PDF bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full PDF binary payload.\n - Raise retrieval-specific errors on failure.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.pdf.scraper.BasePDFClient.fetch", + "signature": "fetch(source: Any)", + "docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF location, such as a file path, object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." + } + } + }, + "PDFScraper": { + "name": "PDFScraper", + "kind": "class", + "path": "omniread.pdf.scraper.PDFScraper", + "signature": "PDFScraper(*, client: BasePDFClient)", + "docstring": "Scraper for PDF sources.\n\nNotes:\n **Responsibilities:**\n\n - Delegates byte retrieval to a PDF client and normalizes output\n into `Content`.\n - Preserves caller-provided metadata.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.pdf.scraper.PDFScraper.fetch", + "signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None) -> Content", + "docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client." + } + } + } + } + } + } + }, + "xlsx": { + "name": "xlsx", + "kind": "module", + "path": "omniread.xlsx", + "signature": null, + "docstring": "# Summary\n\nXLSX subpackage for OmniRead.\n\nProvides acquisition and parsing of Office Open XML spreadsheet (xlsx)\ncontent:\n\n- `BaseXlsxClient`: abstract backing-store client for xlsx bytes.\n- `FileSystemXlsxClient`: local filesystem implementation.\n- `XlsxScraper`: wraps fetched bytes into canonical `Content`.\n- `XlsxParserBase`: content-type-enforcing parser contract.\n- `XlsxParser`: generic string-row parser built on openpyxl.", + "members": { + "BaseXlsxClient": { + "name": "BaseXlsxClient", + "kind": "class", + "path": "omniread.xlsx.BaseXlsxClient", + "signature": null, + "docstring": "Abstract client responsible for retrieving spreadsheet bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full xlsx binary payload.\n - Raise retrieval-specific errors on failure.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.xlsx.BaseXlsxClient.fetch", + "signature": "fetch(source: Any)", + "docstring": "Fetch raw xlsx bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." + } + } + }, + "FileSystemXlsxClient": { + "name": "FileSystemXlsxClient", + "kind": "class", + "path": "omniread.xlsx.FileSystemXlsxClient", + "signature": null, + "docstring": "XLSX client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads spreadsheet files directly from the disk and\n returns their raw binary contents.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.xlsx.FileSystemXlsxClient.fetch", + "signature": "fetch(path: Path)", + "docstring": "Read an xlsx file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the spreadsheet file.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." + } + } + }, + "XlsxParser": { + "name": "XlsxParser", + "kind": "class", + "path": "omniread.xlsx.XlsxParser", + "signature": "XlsxParser(content: Content, *, data_only: bool = True, read_only: bool = True)", + "docstring": "Generic xlsx parser producing string rows from a worksheet.\n\nNotes:\n **Responsibilities:**\n\n - Lazily load the workbook owned by the parser's content.\n - Normalize cells (including dates and numeric values) into\n deterministic string representations.\n - Expose sheet discovery and row extraction helpers.\n\n **Constraints:**\n\n - Cells are rendered with ``str(value)`` after trimming; date and\n datetime values are rendered in ISO format. Consumers requiring\n locale-specific formatting must convert on their side.", + "members": { + "workbook": { + "name": "workbook", + "kind": "attribute", + "path": "omniread.xlsx.XlsxParser.workbook", + "signature": null, + "docstring": "The lazily loaded workbook backing this parser's content." + }, + "sheet_names": { + "name": "sheet_names", + "kind": "attribute", + "path": "omniread.xlsx.XlsxParser.sheet_names", + "signature": null, + "docstring": "Names of all worksheets contained in the workbook." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.xlsx.XlsxParser.parse", + "signature": "parse()", + "docstring": "Parse the first worksheet into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the default (first) worksheet." + }, + "rows": { + "name": "rows", + "kind": "function", + "path": "omniread.xlsx.XlsxParser.rows", + "signature": "rows(sheet: int | str | None = None, *, skip_empty: bool = True)", + "docstring": "Extract normalized string rows from a worksheet.\n\nArgs:\n sheet (int | str | None):\n Worksheet index or title; defaults to the first worksheet.\n skip_empty (bool):\n When True (default), rows whose cells are all blank are omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row.\n\nRaises:\n ValueError:\n If the requested sheet does not exist." + } + } + }, + "XlsxParserBase": { + "name": "XlsxParserBase", + "kind": "class", + "path": "omniread.xlsx.XlsxParserBase", + "signature": null, + "docstring": "Base xlsx parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces xlsx content-type compatibility and provides\n the extension point for implementing concrete xlsx parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.xlsx.XlsxParserBase.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser (XLSX only)." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.xlsx.XlsxParserBase.parse", + "signature": "parse()", + "docstring": "Parse xlsx content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation." + } + } + }, + "XlsxScraper": { + "name": "XlsxScraper", + "kind": "class", + "path": "omniread.xlsx.XlsxScraper", + "signature": "XlsxScraper(*, client: BaseXlsxClient)", + "docstring": "Scraper for xlsx spreadsheet documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw xlsx bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n XLSX content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.xlsx.XlsxScraper.fetch", + "signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None)", + "docstring": "Fetch an xlsx document from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw xlsx bytes, source\n identifier, XLSX content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client." + } + } + }, + "client": { + "name": "client", + "kind": "module", + "path": "omniread.xlsx.client", + "signature": null, + "docstring": "# Summary\n\nXLSX client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw\nOffice Open XML spreadsheet bytes from a concrete backing store.\n\nClients provide low-level access to xlsx binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems", + "members": { + "BaseXlsxClient": { + "name": "BaseXlsxClient", + "kind": "class", + "path": "omniread.xlsx.client.BaseXlsxClient", + "signature": null, + "docstring": "Abstract client responsible for retrieving spreadsheet bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full xlsx binary payload.\n - Raise retrieval-specific errors on failure.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.xlsx.client.BaseXlsxClient.fetch", + "signature": "fetch(source: Any) -> bytes", + "docstring": "Fetch raw xlsx bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." + } + } + }, + "FileSystemXlsxClient": { + "name": "FileSystemXlsxClient", + "kind": "class", + "path": "omniread.xlsx.client.FileSystemXlsxClient", + "signature": null, + "docstring": "XLSX client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads spreadsheet files directly from the disk and\n returns their raw binary contents.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.xlsx.client.FileSystemXlsxClient.fetch", + "signature": "fetch(path: Path) -> bytes", + "docstring": "Read an xlsx file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the spreadsheet file.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." + } + } + } + } + }, + "parser": { + "name": "parser", + "kind": "module", + "path": "omniread.xlsx.parser", + "signature": null, + "docstring": "# Summary\n\nXLSX parser implementations for OmniRead.\n\nThis module provides a concrete, generic parser for Office Open XML\nspreadsheets. It exposes workbook sheets as lists of string rows so\ndownstream consumers can interpret tabular content without depending on\nopenpyxl directly.\n\nThe parser is intentionally statement-agnostic: it performs no header\ndetection or column interpretation beyond basic cell normalization.", + "members": { + "Content": { + "name": "Content", + "kind": "class", + "path": "omniread.xlsx.parser.Content", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", + "members": { + "raw": { + "name": "raw", + "kind": "attribute", + "path": "omniread.xlsx.parser.Content.raw", + "signature": null, + "docstring": "Raw content bytes as retrieved from the source." + }, + "source": { + "name": "source", + "kind": "attribute", + "path": "omniread.xlsx.parser.Content.source", + "signature": null, + "docstring": "Identifier of the content origin (URL, file path, or logical name)." + }, + "content_type": { + "name": "content_type", + "kind": "attribute", + "path": "omniread.xlsx.parser.Content.content_type", + "signature": null, + "docstring": "Optional MIME type of the content, if known." + }, + "metadata": { + "name": "metadata", + "kind": "attribute", + "path": "omniread.xlsx.parser.Content.metadata", + "signature": null, + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." + } + } + }, + "XlsxParserBase": { + "name": "XlsxParserBase", + "kind": "class", + "path": "omniread.xlsx.parser.XlsxParserBase", + "signature": null, + "docstring": "Base xlsx parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces xlsx content-type compatibility and provides\n the extension point for implementing concrete xlsx parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.xlsx.parser.XlsxParserBase.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser (XLSX only)." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.xlsx.parser.XlsxParserBase.parse", + "signature": "parse()", + "docstring": "Parse xlsx content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation." + } + } + }, + "XlsxParser": { + "name": "XlsxParser", + "kind": "class", + "path": "omniread.xlsx.parser.XlsxParser", + "signature": "XlsxParser(content: Content, *, data_only: bool = True, read_only: bool = True)", + "docstring": "Generic xlsx parser producing string rows from a worksheet.\n\nNotes:\n **Responsibilities:**\n\n - Lazily load the workbook owned by the parser's content.\n - Normalize cells (including dates and numeric values) into\n deterministic string representations.\n - Expose sheet discovery and row extraction helpers.\n\n **Constraints:**\n\n - Cells are rendered with ``str(value)`` after trimming; date and\n datetime values are rendered in ISO format. Consumers requiring\n locale-specific formatting must convert on their side.", + "members": { + "workbook": { + "name": "workbook", + "kind": "attribute", + "path": "omniread.xlsx.parser.XlsxParser.workbook", + "signature": null, + "docstring": "The lazily loaded workbook backing this parser's content." + }, + "sheet_names": { + "name": "sheet_names", + "kind": "attribute", + "path": "omniread.xlsx.parser.XlsxParser.sheet_names", + "signature": null, + "docstring": "Names of all worksheets contained in the workbook." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.xlsx.parser.XlsxParser.parse", + "signature": "parse() -> list[list[str]]", + "docstring": "Parse the first worksheet into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the default (first) worksheet." + }, + "rows": { + "name": "rows", + "kind": "function", + "path": "omniread.xlsx.parser.XlsxParser.rows", + "signature": "rows(sheet: int | str | None = None, *, skip_empty: bool = True) -> list[list[str]]", + "docstring": "Extract normalized string rows from a worksheet.\n\nArgs:\n sheet (int | str | None):\n Worksheet index or title; defaults to the first worksheet.\n skip_empty (bool):\n When True (default), rows whose cells are all blank are omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row.\n\nRaises:\n ValueError:\n If the requested sheet does not exist." + } + } + } + } + }, + "parser_base": { + "name": "parser_base", + "kind": "module", + "path": "omniread.xlsx.parser_base", + "signature": null, + "docstring": "# Summary\n\nXLSX parser base implementation for OmniRead.\n\nThis module defines the **XLSX-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for Office Open\nXML spreadsheet content.", + "members": { + "ContentType": { + "name": "ContentType", + "kind": "class", + "path": "omniread.xlsx.parser_base.ContentType", + "signature": null, + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", + "members": { + "HTML": { + "name": "HTML", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.ContentType.HTML", + "signature": null, + "docstring": "HTML document content." + }, + "PDF": { + "name": "PDF", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.ContentType.PDF", + "signature": null, + "docstring": "PDF document content." + }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, + "JSON": { + "name": "JSON", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.ContentType.JSON", + "signature": null, + "docstring": "JSON document content." + }, + "XML": { + "name": "XML", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.ContentType.XML", + "signature": null, + "docstring": "XML document content." + } + } + }, + "BaseParser": { + "name": "BaseParser", + "kind": "class", + "path": "omniread.xlsx.parser_base.BaseParser", + "signature": "BaseParser(content: Content)", + "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.BaseParser.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." + }, + "content": { + "name": "content", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.BaseParser.content", + "signature": null, + "docstring": null + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.xlsx.parser_base.BaseParser.parse", + "signature": "parse()", + "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." + }, + "supports": { + "name": "supports", + "kind": "function", + "path": "omniread.xlsx.parser_base.BaseParser.supports", + "signature": "supports()", + "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." + } + } + }, + "T": { + "name": "T", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.T", + "signature": null, + "docstring": null + }, + "XlsxParserBase": { + "name": "XlsxParserBase", + "kind": "class", + "path": "omniread.xlsx.parser_base.XlsxParserBase", + "signature": null, + "docstring": "Base xlsx parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces xlsx content-type compatibility and provides\n the extension point for implementing concrete xlsx parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.XlsxParserBase.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser (XLSX only)." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.xlsx.parser_base.XlsxParserBase.parse", + "signature": "parse() -> T", + "docstring": "Parse xlsx content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation." + } + } + } + } + }, + "scraper": { + "name": "scraper", + "kind": "module", + "path": "omniread.xlsx.scraper", + "signature": null, + "docstring": "# Summary\n\nXLSX scraper for OmniRead.\n\nThis module defines the scraper responsible for acquiring raw Office Open\nXML spreadsheet content from a backing store via a configured client.\n\nThe scraper does not interpret or parse the acquired bytes; it wraps them in\nthe canonical `Content` model.", + "members": { + "Content": { + "name": "Content", + "kind": "class", + "path": "omniread.xlsx.scraper.Content", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", + "members": { + "raw": { + "name": "raw", + "kind": "attribute", + "path": "omniread.xlsx.scraper.Content.raw", + "signature": null, + "docstring": "Raw content bytes as retrieved from the source." + }, + "source": { + "name": "source", + "kind": "attribute", + "path": "omniread.xlsx.scraper.Content.source", + "signature": null, + "docstring": "Identifier of the content origin (URL, file path, or logical name)." + }, + "content_type": { + "name": "content_type", + "kind": "attribute", + "path": "omniread.xlsx.scraper.Content.content_type", + "signature": null, + "docstring": "Optional MIME type of the content, if known." + }, + "metadata": { + "name": "metadata", + "kind": "attribute", + "path": "omniread.xlsx.scraper.Content.metadata", + "signature": null, + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." + } + } + }, + "ContentType": { + "name": "ContentType", + "kind": "class", + "path": "omniread.xlsx.scraper.ContentType", + "signature": null, + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", + "members": { + "HTML": { + "name": "HTML", + "kind": "attribute", + "path": "omniread.xlsx.scraper.ContentType.HTML", + "signature": null, + "docstring": "HTML document content." + }, + "PDF": { + "name": "PDF", + "kind": "attribute", + "path": "omniread.xlsx.scraper.ContentType.PDF", + "signature": null, + "docstring": "PDF document content." + }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.xlsx.scraper.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.xlsx.scraper.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, + "JSON": { + "name": "JSON", + "kind": "attribute", + "path": "omniread.xlsx.scraper.ContentType.JSON", + "signature": null, + "docstring": "JSON document content." + }, + "XML": { + "name": "XML", + "kind": "attribute", + "path": "omniread.xlsx.scraper.ContentType.XML", + "signature": null, + "docstring": "XML document content." + } + } + }, + "BaseXlsxClient": { + "name": "BaseXlsxClient", + "kind": "class", + "path": "omniread.xlsx.scraper.BaseXlsxClient", + "signature": null, + "docstring": "Abstract client responsible for retrieving spreadsheet bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full xlsx binary payload.\n - Raise retrieval-specific errors on failure.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.xlsx.scraper.BaseXlsxClient.fetch", + "signature": "fetch(source: Any)", + "docstring": "Fetch raw xlsx bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." + } + } + }, + "XlsxScraper": { + "name": "XlsxScraper", + "kind": "class", + "path": "omniread.xlsx.scraper.XlsxScraper", + "signature": "XlsxScraper(*, client: BaseXlsxClient)", + "docstring": "Scraper for xlsx spreadsheet documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw xlsx bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n XLSX content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.xlsx.scraper.XlsxScraper.fetch", + "signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None) -> Content", + "docstring": "Fetch an xlsx document from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw xlsx bytes, source\n identifier, XLSX content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client." + } + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/mcp_docs/modules/omniread.pdf.client.json b/docs/mcp/modules/omniread.pdf.client.json similarity index 66% rename from mcp_docs/modules/omniread.pdf.client.json rename to docs/mcp/modules/omniread.pdf.client.json index d7e0484..b73e277 100644 --- a/mcp_docs/modules/omniread.pdf.client.json +++ b/docs/mcp/modules/omniread.pdf.client.json @@ -4,46 +4,18 @@ "path": "omniread.pdf.client", "docstring": "# Summary\n\nPDF client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw PDF\nbytes from a concrete backing store.\n\nClients provide low-level access to PDF binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems", "objects": { - "Any": { - "name": "Any", - "kind": "alias", - "path": "omniread.pdf.client.Any", - "signature": "", - "docstring": null - }, - "ABC": { - "name": "ABC", - "kind": "alias", - "path": "omniread.pdf.client.ABC", - "signature": "", - "docstring": null - }, - "abstractmethod": { - "name": "abstractmethod", - "kind": "alias", - "path": "omniread.pdf.client.abstractmethod", - "signature": "", - "docstring": null - }, - "Path": { - "name": "Path", - "kind": "alias", - "path": "omniread.pdf.client.Path", - "signature": "", - "docstring": null - }, "BasePDFClient": { "name": "BasePDFClient", "kind": "class", "path": "omniread.pdf.client.BasePDFClient", - "signature": "", + "signature": null, "docstring": "Abstract client responsible for retrieving PDF bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full PDF binary payload.\n - Raise retrieval-specific errors on failure.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.client.BasePDFClient.fetch", - "signature": "", + "signature": "fetch(source: Any) -> bytes", "docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF location, such as a file path, object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." } } @@ -52,14 +24,14 @@ "name": "FileSystemPDFClient", "kind": "class", "path": "omniread.pdf.client.FileSystemPDFClient", - "signature": "", + "signature": null, "docstring": "PDF client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads PDF files directly from the disk and returns\n their raw binary contents.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.client.FileSystemPDFClient.fetch", - "signature": "", + "signature": "fetch(path: Path) -> bytes", "docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the PDF file.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." } } diff --git a/mcp_docs/modules/omniread.pdf.json b/docs/mcp/modules/omniread.pdf.json similarity index 69% rename from mcp_docs/modules/omniread.pdf.json rename to docs/mcp/modules/omniread.pdf.json index ca41c24..1695b44 100644 --- a/mcp_docs/modules/omniread.pdf.json +++ b/docs/mcp/modules/omniread.pdf.json @@ -8,14 +8,14 @@ "name": "FileSystemPDFClient", "kind": "class", "path": "omniread.pdf.FileSystemPDFClient", - "signature": "", + "signature": null, "docstring": "PDF client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads PDF files directly from the disk and returns\n their raw binary contents.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.FileSystemPDFClient.fetch", - "signature": "", + "signature": "fetch(path: Path)", "docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the PDF file.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." } } @@ -24,15 +24,15 @@ "name": "PDFScraper", "kind": "class", "path": "omniread.pdf.PDFScraper", - "signature": "", + "signature": "PDFScraper(*, client: BasePDFClient)", "docstring": "Scraper for PDF sources.\n\nNotes:\n **Responsibilities:**\n\n - Delegates byte retrieval to a PDF client and normalizes output\n into `Content`.\n - Preserves caller-provided metadata.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.PDFScraper.fetch", - "signature": "", - "docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client." + "signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None)", + "docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client." } } }, @@ -40,21 +40,21 @@ "name": "PDFParser", "kind": "class", "path": "omniread.pdf.PDFParser", - "signature": "", + "signature": null, "docstring": "Base PDF parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces PDF content-type compatibility and provides\n the extension point for implementing concrete PDF parsing strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", "members": { "supported_types": { "name": "supported_types", "kind": "attribute", "path": "omniread.pdf.PDFParser.supported_types", - "signature": "", + "signature": null, "docstring": "Set of content types supported by this parser (PDF only)." }, "parse": { "name": "parse", "kind": "function", "path": "omniread.pdf.PDFParser.parse", - "signature": "", + "signature": "parse()", "docstring": "Parse PDF content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the PDF binary payload and\n return a deterministic, structured output." } } @@ -66,46 +66,18 @@ "signature": null, "docstring": "# Summary\n\nPDF client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw PDF\nbytes from a concrete backing store.\n\nClients provide low-level access to PDF binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems", "members": { - "Any": { - "name": "Any", - "kind": "alias", - "path": "omniread.pdf.client.Any", - "signature": "", - "docstring": null - }, - "ABC": { - "name": "ABC", - "kind": "alias", - "path": "omniread.pdf.client.ABC", - "signature": "", - "docstring": null - }, - "abstractmethod": { - "name": "abstractmethod", - "kind": "alias", - "path": "omniread.pdf.client.abstractmethod", - "signature": "", - "docstring": null - }, - "Path": { - "name": "Path", - "kind": "alias", - "path": "omniread.pdf.client.Path", - "signature": "", - "docstring": null - }, "BasePDFClient": { "name": "BasePDFClient", "kind": "class", "path": "omniread.pdf.client.BasePDFClient", - "signature": "", + "signature": null, "docstring": "Abstract client responsible for retrieving PDF bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full PDF binary payload.\n - Raise retrieval-specific errors on failure.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.client.BasePDFClient.fetch", - "signature": "", + "signature": "fetch(source: Any) -> bytes", "docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF location, such as a file path, object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." } } @@ -114,14 +86,14 @@ "name": "FileSystemPDFClient", "kind": "class", "path": "omniread.pdf.client.FileSystemPDFClient", - "signature": "", + "signature": null, "docstring": "PDF client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads PDF files directly from the disk and returns\n their raw binary contents.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.client.FileSystemPDFClient.fetch", - "signature": "", + "signature": "fetch(path: Path) -> bytes", "docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the PDF file.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." } } @@ -135,60 +107,53 @@ "signature": null, "docstring": "# Summary\n\nPDF parser base implementations for OmniRead.\n\nThis module defines the **PDF-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for PDF content.\n\nPDF parsers are responsible for interpreting binary PDF data and producing\nstructured representations suitable for downstream consumption.", "members": { - "Generic": { - "name": "Generic", - "kind": "alias", - "path": "omniread.pdf.parser.Generic", - "signature": "", - "docstring": null - }, - "TypeVar": { - "name": "TypeVar", - "kind": "alias", - "path": "omniread.pdf.parser.TypeVar", - "signature": "", - "docstring": null - }, - "abstractmethod": { - "name": "abstractmethod", - "kind": "alias", - "path": "omniread.pdf.parser.abstractmethod", - "signature": "", - "docstring": null - }, "ContentType": { "name": "ContentType", "kind": "class", "path": "omniread.pdf.parser.ContentType", - "signature": "", + "signature": null, "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", "kind": "attribute", "path": "omniread.pdf.parser.ContentType.HTML", - "signature": "", + "signature": null, "docstring": "HTML document content." }, "PDF": { "name": "PDF", "kind": "attribute", "path": "omniread.pdf.parser.ContentType.PDF", - "signature": "", + "signature": null, "docstring": "PDF document content." }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.pdf.parser.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.pdf.parser.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, "JSON": { "name": "JSON", "kind": "attribute", "path": "omniread.pdf.parser.ContentType.JSON", - "signature": "", + "signature": null, "docstring": "JSON document content." }, "XML": { "name": "XML", "kind": "attribute", "path": "omniread.pdf.parser.ContentType.XML", - "signature": "", + "signature": null, "docstring": "XML document content." } } @@ -197,35 +162,35 @@ "name": "BaseParser", "kind": "class", "path": "omniread.pdf.parser.BaseParser", - "signature": "", + "signature": "BaseParser(content: Content)", "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", "members": { "supported_types": { "name": "supported_types", "kind": "attribute", "path": "omniread.pdf.parser.BaseParser.supported_types", - "signature": "", + "signature": null, "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." }, "content": { "name": "content", "kind": "attribute", "path": "omniread.pdf.parser.BaseParser.content", - "signature": "", + "signature": null, "docstring": null }, "parse": { "name": "parse", "kind": "function", "path": "omniread.pdf.parser.BaseParser.parse", - "signature": "", + "signature": "parse()", "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." }, "supports": { "name": "supports", "kind": "function", "path": "omniread.pdf.parser.BaseParser.supports", - "signature": "", + "signature": "supports()", "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." } } @@ -241,7 +206,7 @@ "name": "PDFParser", "kind": "class", "path": "omniread.pdf.parser.PDFParser", - "signature": "", + "signature": null, "docstring": "Base PDF parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces PDF content-type compatibility and provides\n the extension point for implementing concrete PDF parsing strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", "members": { "supported_types": { @@ -255,7 +220,7 @@ "name": "parse", "kind": "function", "path": "omniread.pdf.parser.PDFParser.parse", - "signature": "", + "signature": "parse() -> T", "docstring": "Parse PDF content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the PDF binary payload and\n return a deterministic, structured output." } } @@ -269,60 +234,39 @@ "signature": null, "docstring": "# Summary\n\nPDF scraping implementation for OmniRead.\n\nThis module provides a PDF-specific scraper that coordinates PDF byte\nretrieval via a client and normalizes the result into a `Content` object.\n\nThe scraper implements the core `BaseScraper` contract while delegating\nall storage and access concerns to a `BasePDFClient` implementation.", "members": { - "Any": { - "name": "Any", - "kind": "alias", - "path": "omniread.pdf.scraper.Any", - "signature": "", - "docstring": null - }, - "Mapping": { - "name": "Mapping", - "kind": "alias", - "path": "omniread.pdf.scraper.Mapping", - "signature": "", - "docstring": null - }, - "Optional": { - "name": "Optional", - "kind": "alias", - "path": "omniread.pdf.scraper.Optional", - "signature": "", - "docstring": null - }, "Content": { "name": "Content", "kind": "class", "path": "omniread.pdf.scraper.Content", - "signature": "", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.pdf.scraper.Content.raw", - "signature": "", + "signature": null, "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.pdf.scraper.Content.source", - "signature": "", + "signature": null, "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.pdf.scraper.Content.content_type", - "signature": "", + "signature": null, "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.pdf.scraper.Content.metadata", - "signature": "", + "signature": null, "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } @@ -331,35 +275,49 @@ "name": "ContentType", "kind": "class", "path": "omniread.pdf.scraper.ContentType", - "signature": "", + "signature": null, "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", "kind": "attribute", "path": "omniread.pdf.scraper.ContentType.HTML", - "signature": "", + "signature": null, "docstring": "HTML document content." }, "PDF": { "name": "PDF", "kind": "attribute", "path": "omniread.pdf.scraper.ContentType.PDF", - "signature": "", + "signature": null, "docstring": "PDF document content." }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.pdf.scraper.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.pdf.scraper.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, "JSON": { "name": "JSON", "kind": "attribute", "path": "omniread.pdf.scraper.ContentType.JSON", - "signature": "", + "signature": null, "docstring": "JSON document content." }, "XML": { "name": "XML", "kind": "attribute", "path": "omniread.pdf.scraper.ContentType.XML", - "signature": "", + "signature": null, "docstring": "XML document content." } } @@ -368,15 +326,15 @@ "name": "BaseScraper", "kind": "class", "path": "omniread.pdf.scraper.BaseScraper", - "signature": "", + "signature": null, "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.scraper.BaseScraper.fetch", - "signature": "", - "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." + "signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None)", + "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." } } }, @@ -384,14 +342,14 @@ "name": "BasePDFClient", "kind": "class", "path": "omniread.pdf.scraper.BasePDFClient", - "signature": "", + "signature": null, "docstring": "Abstract client responsible for retrieving PDF bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full PDF binary payload.\n - Raise retrieval-specific errors on failure.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.scraper.BasePDFClient.fetch", - "signature": "", + "signature": "fetch(source: Any)", "docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF location, such as a file path, object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." } } @@ -400,15 +358,15 @@ "name": "PDFScraper", "kind": "class", "path": "omniread.pdf.scraper.PDFScraper", - "signature": "", + "signature": "PDFScraper(*, client: BasePDFClient)", "docstring": "Scraper for PDF sources.\n\nNotes:\n **Responsibilities:**\n\n - Delegates byte retrieval to a PDF client and normalizes output\n into `Content`.\n - Preserves caller-provided metadata.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.scraper.PDFScraper.fetch", - "signature": "", - "docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client." + "signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None) -> Content", + "docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client." } } } diff --git a/mcp_docs/modules/omniread.pdf.parser.json b/docs/mcp/modules/omniread.pdf.parser.json similarity index 72% rename from mcp_docs/modules/omniread.pdf.parser.json rename to docs/mcp/modules/omniread.pdf.parser.json index e4fccb5..e330994 100644 --- a/mcp_docs/modules/omniread.pdf.parser.json +++ b/docs/mcp/modules/omniread.pdf.parser.json @@ -4,60 +4,53 @@ "path": "omniread.pdf.parser", "docstring": "# Summary\n\nPDF parser base implementations for OmniRead.\n\nThis module defines the **PDF-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for PDF content.\n\nPDF parsers are responsible for interpreting binary PDF data and producing\nstructured representations suitable for downstream consumption.", "objects": { - "Generic": { - "name": "Generic", - "kind": "alias", - "path": "omniread.pdf.parser.Generic", - "signature": "", - "docstring": null - }, - "TypeVar": { - "name": "TypeVar", - "kind": "alias", - "path": "omniread.pdf.parser.TypeVar", - "signature": "", - "docstring": null - }, - "abstractmethod": { - "name": "abstractmethod", - "kind": "alias", - "path": "omniread.pdf.parser.abstractmethod", - "signature": "", - "docstring": null - }, "ContentType": { "name": "ContentType", "kind": "class", "path": "omniread.pdf.parser.ContentType", - "signature": "", + "signature": null, "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", "kind": "attribute", "path": "omniread.pdf.parser.ContentType.HTML", - "signature": "", + "signature": null, "docstring": "HTML document content." }, "PDF": { "name": "PDF", "kind": "attribute", "path": "omniread.pdf.parser.ContentType.PDF", - "signature": "", + "signature": null, "docstring": "PDF document content." }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.pdf.parser.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.pdf.parser.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, "JSON": { "name": "JSON", "kind": "attribute", "path": "omniread.pdf.parser.ContentType.JSON", - "signature": "", + "signature": null, "docstring": "JSON document content." }, "XML": { "name": "XML", "kind": "attribute", "path": "omniread.pdf.parser.ContentType.XML", - "signature": "", + "signature": null, "docstring": "XML document content." } } @@ -66,35 +59,35 @@ "name": "BaseParser", "kind": "class", "path": "omniread.pdf.parser.BaseParser", - "signature": "", + "signature": "BaseParser(content: Content)", "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", "members": { "supported_types": { "name": "supported_types", "kind": "attribute", "path": "omniread.pdf.parser.BaseParser.supported_types", - "signature": "", + "signature": null, "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." }, "content": { "name": "content", "kind": "attribute", "path": "omniread.pdf.parser.BaseParser.content", - "signature": "", + "signature": null, "docstring": null }, "parse": { "name": "parse", "kind": "function", "path": "omniread.pdf.parser.BaseParser.parse", - "signature": "", + "signature": "parse()", "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." }, "supports": { "name": "supports", "kind": "function", "path": "omniread.pdf.parser.BaseParser.supports", - "signature": "", + "signature": "supports()", "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." } } @@ -110,7 +103,7 @@ "name": "PDFParser", "kind": "class", "path": "omniread.pdf.parser.PDFParser", - "signature": "", + "signature": null, "docstring": "Base PDF parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces PDF content-type compatibility and provides\n the extension point for implementing concrete PDF parsing strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", "members": { "supported_types": { @@ -124,7 +117,7 @@ "name": "parse", "kind": "function", "path": "omniread.pdf.parser.PDFParser.parse", - "signature": "", + "signature": "parse() -> T", "docstring": "Parse PDF content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the PDF binary payload and\n return a deterministic, structured output." } } diff --git a/mcp_docs/modules/omniread.pdf.scraper.json b/docs/mcp/modules/omniread.pdf.scraper.json similarity index 65% rename from mcp_docs/modules/omniread.pdf.scraper.json rename to docs/mcp/modules/omniread.pdf.scraper.json index 23de5a1..ff03d34 100644 --- a/mcp_docs/modules/omniread.pdf.scraper.json +++ b/docs/mcp/modules/omniread.pdf.scraper.json @@ -4,60 +4,39 @@ "path": "omniread.pdf.scraper", "docstring": "# Summary\n\nPDF scraping implementation for OmniRead.\n\nThis module provides a PDF-specific scraper that coordinates PDF byte\nretrieval via a client and normalizes the result into a `Content` object.\n\nThe scraper implements the core `BaseScraper` contract while delegating\nall storage and access concerns to a `BasePDFClient` implementation.", "objects": { - "Any": { - "name": "Any", - "kind": "alias", - "path": "omniread.pdf.scraper.Any", - "signature": "", - "docstring": null - }, - "Mapping": { - "name": "Mapping", - "kind": "alias", - "path": "omniread.pdf.scraper.Mapping", - "signature": "", - "docstring": null - }, - "Optional": { - "name": "Optional", - "kind": "alias", - "path": "omniread.pdf.scraper.Optional", - "signature": "", - "docstring": null - }, "Content": { "name": "Content", "kind": "class", "path": "omniread.pdf.scraper.Content", - "signature": "", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.pdf.scraper.Content.raw", - "signature": "", + "signature": null, "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.pdf.scraper.Content.source", - "signature": "", + "signature": null, "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.pdf.scraper.Content.content_type", - "signature": "", + "signature": null, "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.pdf.scraper.Content.metadata", - "signature": "", + "signature": null, "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } @@ -66,35 +45,49 @@ "name": "ContentType", "kind": "class", "path": "omniread.pdf.scraper.ContentType", - "signature": "", + "signature": null, "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", "kind": "attribute", "path": "omniread.pdf.scraper.ContentType.HTML", - "signature": "", + "signature": null, "docstring": "HTML document content." }, "PDF": { "name": "PDF", "kind": "attribute", "path": "omniread.pdf.scraper.ContentType.PDF", - "signature": "", + "signature": null, "docstring": "PDF document content." }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.pdf.scraper.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.pdf.scraper.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, "JSON": { "name": "JSON", "kind": "attribute", "path": "omniread.pdf.scraper.ContentType.JSON", - "signature": "", + "signature": null, "docstring": "JSON document content." }, "XML": { "name": "XML", "kind": "attribute", "path": "omniread.pdf.scraper.ContentType.XML", - "signature": "", + "signature": null, "docstring": "XML document content." } } @@ -103,15 +96,15 @@ "name": "BaseScraper", "kind": "class", "path": "omniread.pdf.scraper.BaseScraper", - "signature": "", + "signature": null, "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.scraper.BaseScraper.fetch", - "signature": "", - "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." + "signature": "fetch(source: str, *, metadata: Mapping[str, Any] | None = None)", + "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Mapping[str, Any] | None, optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." } } }, @@ -119,14 +112,14 @@ "name": "BasePDFClient", "kind": "class", "path": "omniread.pdf.scraper.BasePDFClient", - "signature": "", + "signature": null, "docstring": "Abstract client responsible for retrieving PDF bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full PDF binary payload.\n - Raise retrieval-specific errors on failure.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.scraper.BasePDFClient.fetch", - "signature": "", + "signature": "fetch(source: Any)", "docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF location, such as a file path, object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." } } @@ -135,15 +128,15 @@ "name": "PDFScraper", "kind": "class", "path": "omniread.pdf.scraper.PDFScraper", - "signature": "", + "signature": "PDFScraper(*, client: BasePDFClient)", "docstring": "Scraper for PDF sources.\n\nNotes:\n **Responsibilities:**\n\n - Delegates byte retrieval to a PDF client and normalizes output\n into `Content`.\n - Preserves caller-provided metadata.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.scraper.PDFScraper.fetch", - "signature": "", - "docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client." + "signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None) -> Content", + "docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client." } } } diff --git a/docs/mcp/modules/omniread.xlsx.client.json b/docs/mcp/modules/omniread.xlsx.client.json new file mode 100644 index 0000000..0234f37 --- /dev/null +++ b/docs/mcp/modules/omniread.xlsx.client.json @@ -0,0 +1,41 @@ +{ + "module": "omniread.xlsx.client", + "content": { + "path": "omniread.xlsx.client", + "docstring": "# Summary\n\nXLSX client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw\nOffice Open XML spreadsheet bytes from a concrete backing store.\n\nClients provide low-level access to xlsx binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems", + "objects": { + "BaseXlsxClient": { + "name": "BaseXlsxClient", + "kind": "class", + "path": "omniread.xlsx.client.BaseXlsxClient", + "signature": null, + "docstring": "Abstract client responsible for retrieving spreadsheet bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full xlsx binary payload.\n - Raise retrieval-specific errors on failure.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.xlsx.client.BaseXlsxClient.fetch", + "signature": "fetch(source: Any) -> bytes", + "docstring": "Fetch raw xlsx bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." + } + } + }, + "FileSystemXlsxClient": { + "name": "FileSystemXlsxClient", + "kind": "class", + "path": "omniread.xlsx.client.FileSystemXlsxClient", + "signature": null, + "docstring": "XLSX client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads spreadsheet files directly from the disk and\n returns their raw binary contents.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.xlsx.client.FileSystemXlsxClient.fetch", + "signature": "fetch(path: Path) -> bytes", + "docstring": "Read an xlsx file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the spreadsheet file.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." + } + } + } + } + } +} \ No newline at end of file diff --git a/docs/mcp/modules/omniread.xlsx.json b/docs/mcp/modules/omniread.xlsx.json new file mode 100644 index 0000000..ef391a2 --- /dev/null +++ b/docs/mcp/modules/omniread.xlsx.json @@ -0,0 +1,520 @@ +{ + "module": "omniread.xlsx", + "content": { + "path": "omniread.xlsx", + "docstring": "# Summary\n\nXLSX subpackage for OmniRead.\n\nProvides acquisition and parsing of Office Open XML spreadsheet (xlsx)\ncontent:\n\n- `BaseXlsxClient`: abstract backing-store client for xlsx bytes.\n- `FileSystemXlsxClient`: local filesystem implementation.\n- `XlsxScraper`: wraps fetched bytes into canonical `Content`.\n- `XlsxParserBase`: content-type-enforcing parser contract.\n- `XlsxParser`: generic string-row parser built on openpyxl.", + "objects": { + "BaseXlsxClient": { + "name": "BaseXlsxClient", + "kind": "class", + "path": "omniread.xlsx.BaseXlsxClient", + "signature": null, + "docstring": "Abstract client responsible for retrieving spreadsheet bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full xlsx binary payload.\n - Raise retrieval-specific errors on failure.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.xlsx.BaseXlsxClient.fetch", + "signature": "fetch(source: Any)", + "docstring": "Fetch raw xlsx bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." + } + } + }, + "FileSystemXlsxClient": { + "name": "FileSystemXlsxClient", + "kind": "class", + "path": "omniread.xlsx.FileSystemXlsxClient", + "signature": null, + "docstring": "XLSX client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads spreadsheet files directly from the disk and\n returns their raw binary contents.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.xlsx.FileSystemXlsxClient.fetch", + "signature": "fetch(path: Path)", + "docstring": "Read an xlsx file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the spreadsheet file.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." + } + } + }, + "XlsxParser": { + "name": "XlsxParser", + "kind": "class", + "path": "omniread.xlsx.XlsxParser", + "signature": "XlsxParser(content: Content, *, data_only: bool = True, read_only: bool = True)", + "docstring": "Generic xlsx parser producing string rows from a worksheet.\n\nNotes:\n **Responsibilities:**\n\n - Lazily load the workbook owned by the parser's content.\n - Normalize cells (including dates and numeric values) into\n deterministic string representations.\n - Expose sheet discovery and row extraction helpers.\n\n **Constraints:**\n\n - Cells are rendered with ``str(value)`` after trimming; date and\n datetime values are rendered in ISO format. Consumers requiring\n locale-specific formatting must convert on their side.", + "members": { + "workbook": { + "name": "workbook", + "kind": "attribute", + "path": "omniread.xlsx.XlsxParser.workbook", + "signature": null, + "docstring": "The lazily loaded workbook backing this parser's content." + }, + "sheet_names": { + "name": "sheet_names", + "kind": "attribute", + "path": "omniread.xlsx.XlsxParser.sheet_names", + "signature": null, + "docstring": "Names of all worksheets contained in the workbook." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.xlsx.XlsxParser.parse", + "signature": "parse()", + "docstring": "Parse the first worksheet into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the default (first) worksheet." + }, + "rows": { + "name": "rows", + "kind": "function", + "path": "omniread.xlsx.XlsxParser.rows", + "signature": "rows(sheet: int | str | None = None, *, skip_empty: bool = True)", + "docstring": "Extract normalized string rows from a worksheet.\n\nArgs:\n sheet (int | str | None):\n Worksheet index or title; defaults to the first worksheet.\n skip_empty (bool):\n When True (default), rows whose cells are all blank are omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row.\n\nRaises:\n ValueError:\n If the requested sheet does not exist." + } + } + }, + "XlsxParserBase": { + "name": "XlsxParserBase", + "kind": "class", + "path": "omniread.xlsx.XlsxParserBase", + "signature": null, + "docstring": "Base xlsx parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces xlsx content-type compatibility and provides\n the extension point for implementing concrete xlsx parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.xlsx.XlsxParserBase.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser (XLSX only)." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.xlsx.XlsxParserBase.parse", + "signature": "parse()", + "docstring": "Parse xlsx content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation." + } + } + }, + "XlsxScraper": { + "name": "XlsxScraper", + "kind": "class", + "path": "omniread.xlsx.XlsxScraper", + "signature": "XlsxScraper(*, client: BaseXlsxClient)", + "docstring": "Scraper for xlsx spreadsheet documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw xlsx bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n XLSX content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.xlsx.XlsxScraper.fetch", + "signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None)", + "docstring": "Fetch an xlsx document from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw xlsx bytes, source\n identifier, XLSX content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client." + } + } + }, + "client": { + "name": "client", + "kind": "module", + "path": "omniread.xlsx.client", + "signature": null, + "docstring": "# Summary\n\nXLSX client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw\nOffice Open XML spreadsheet bytes from a concrete backing store.\n\nClients provide low-level access to xlsx binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems", + "members": { + "BaseXlsxClient": { + "name": "BaseXlsxClient", + "kind": "class", + "path": "omniread.xlsx.client.BaseXlsxClient", + "signature": null, + "docstring": "Abstract client responsible for retrieving spreadsheet bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full xlsx binary payload.\n - Raise retrieval-specific errors on failure.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.xlsx.client.BaseXlsxClient.fetch", + "signature": "fetch(source: Any) -> bytes", + "docstring": "Fetch raw xlsx bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." + } + } + }, + "FileSystemXlsxClient": { + "name": "FileSystemXlsxClient", + "kind": "class", + "path": "omniread.xlsx.client.FileSystemXlsxClient", + "signature": null, + "docstring": "XLSX client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads spreadsheet files directly from the disk and\n returns their raw binary contents.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.xlsx.client.FileSystemXlsxClient.fetch", + "signature": "fetch(path: Path) -> bytes", + "docstring": "Read an xlsx file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the spreadsheet file.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." + } + } + } + } + }, + "parser": { + "name": "parser", + "kind": "module", + "path": "omniread.xlsx.parser", + "signature": null, + "docstring": "# Summary\n\nXLSX parser implementations for OmniRead.\n\nThis module provides a concrete, generic parser for Office Open XML\nspreadsheets. It exposes workbook sheets as lists of string rows so\ndownstream consumers can interpret tabular content without depending on\nopenpyxl directly.\n\nThe parser is intentionally statement-agnostic: it performs no header\ndetection or column interpretation beyond basic cell normalization.", + "members": { + "Content": { + "name": "Content", + "kind": "class", + "path": "omniread.xlsx.parser.Content", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", + "members": { + "raw": { + "name": "raw", + "kind": "attribute", + "path": "omniread.xlsx.parser.Content.raw", + "signature": null, + "docstring": "Raw content bytes as retrieved from the source." + }, + "source": { + "name": "source", + "kind": "attribute", + "path": "omniread.xlsx.parser.Content.source", + "signature": null, + "docstring": "Identifier of the content origin (URL, file path, or logical name)." + }, + "content_type": { + "name": "content_type", + "kind": "attribute", + "path": "omniread.xlsx.parser.Content.content_type", + "signature": null, + "docstring": "Optional MIME type of the content, if known." + }, + "metadata": { + "name": "metadata", + "kind": "attribute", + "path": "omniread.xlsx.parser.Content.metadata", + "signature": null, + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." + } + } + }, + "XlsxParserBase": { + "name": "XlsxParserBase", + "kind": "class", + "path": "omniread.xlsx.parser.XlsxParserBase", + "signature": null, + "docstring": "Base xlsx parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces xlsx content-type compatibility and provides\n the extension point for implementing concrete xlsx parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.xlsx.parser.XlsxParserBase.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser (XLSX only)." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.xlsx.parser.XlsxParserBase.parse", + "signature": "parse()", + "docstring": "Parse xlsx content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation." + } + } + }, + "XlsxParser": { + "name": "XlsxParser", + "kind": "class", + "path": "omniread.xlsx.parser.XlsxParser", + "signature": "XlsxParser(content: Content, *, data_only: bool = True, read_only: bool = True)", + "docstring": "Generic xlsx parser producing string rows from a worksheet.\n\nNotes:\n **Responsibilities:**\n\n - Lazily load the workbook owned by the parser's content.\n - Normalize cells (including dates and numeric values) into\n deterministic string representations.\n - Expose sheet discovery and row extraction helpers.\n\n **Constraints:**\n\n - Cells are rendered with ``str(value)`` after trimming; date and\n datetime values are rendered in ISO format. Consumers requiring\n locale-specific formatting must convert on their side.", + "members": { + "workbook": { + "name": "workbook", + "kind": "attribute", + "path": "omniread.xlsx.parser.XlsxParser.workbook", + "signature": null, + "docstring": "The lazily loaded workbook backing this parser's content." + }, + "sheet_names": { + "name": "sheet_names", + "kind": "attribute", + "path": "omniread.xlsx.parser.XlsxParser.sheet_names", + "signature": null, + "docstring": "Names of all worksheets contained in the workbook." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.xlsx.parser.XlsxParser.parse", + "signature": "parse() -> list[list[str]]", + "docstring": "Parse the first worksheet into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the default (first) worksheet." + }, + "rows": { + "name": "rows", + "kind": "function", + "path": "omniread.xlsx.parser.XlsxParser.rows", + "signature": "rows(sheet: int | str | None = None, *, skip_empty: bool = True) -> list[list[str]]", + "docstring": "Extract normalized string rows from a worksheet.\n\nArgs:\n sheet (int | str | None):\n Worksheet index or title; defaults to the first worksheet.\n skip_empty (bool):\n When True (default), rows whose cells are all blank are omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row.\n\nRaises:\n ValueError:\n If the requested sheet does not exist." + } + } + } + } + }, + "parser_base": { + "name": "parser_base", + "kind": "module", + "path": "omniread.xlsx.parser_base", + "signature": null, + "docstring": "# Summary\n\nXLSX parser base implementation for OmniRead.\n\nThis module defines the **XLSX-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for Office Open\nXML spreadsheet content.", + "members": { + "ContentType": { + "name": "ContentType", + "kind": "class", + "path": "omniread.xlsx.parser_base.ContentType", + "signature": null, + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", + "members": { + "HTML": { + "name": "HTML", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.ContentType.HTML", + "signature": null, + "docstring": "HTML document content." + }, + "PDF": { + "name": "PDF", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.ContentType.PDF", + "signature": null, + "docstring": "PDF document content." + }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, + "JSON": { + "name": "JSON", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.ContentType.JSON", + "signature": null, + "docstring": "JSON document content." + }, + "XML": { + "name": "XML", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.ContentType.XML", + "signature": null, + "docstring": "XML document content." + } + } + }, + "BaseParser": { + "name": "BaseParser", + "kind": "class", + "path": "omniread.xlsx.parser_base.BaseParser", + "signature": "BaseParser(content: Content)", + "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.BaseParser.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." + }, + "content": { + "name": "content", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.BaseParser.content", + "signature": null, + "docstring": null + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.xlsx.parser_base.BaseParser.parse", + "signature": "parse()", + "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." + }, + "supports": { + "name": "supports", + "kind": "function", + "path": "omniread.xlsx.parser_base.BaseParser.supports", + "signature": "supports()", + "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." + } + } + }, + "T": { + "name": "T", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.T", + "signature": null, + "docstring": null + }, + "XlsxParserBase": { + "name": "XlsxParserBase", + "kind": "class", + "path": "omniread.xlsx.parser_base.XlsxParserBase", + "signature": null, + "docstring": "Base xlsx parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces xlsx content-type compatibility and provides\n the extension point for implementing concrete xlsx parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.XlsxParserBase.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser (XLSX only)." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.xlsx.parser_base.XlsxParserBase.parse", + "signature": "parse() -> T", + "docstring": "Parse xlsx content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation." + } + } + } + } + }, + "scraper": { + "name": "scraper", + "kind": "module", + "path": "omniread.xlsx.scraper", + "signature": null, + "docstring": "# Summary\n\nXLSX scraper for OmniRead.\n\nThis module defines the scraper responsible for acquiring raw Office Open\nXML spreadsheet content from a backing store via a configured client.\n\nThe scraper does not interpret or parse the acquired bytes; it wraps them in\nthe canonical `Content` model.", + "members": { + "Content": { + "name": "Content", + "kind": "class", + "path": "omniread.xlsx.scraper.Content", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", + "members": { + "raw": { + "name": "raw", + "kind": "attribute", + "path": "omniread.xlsx.scraper.Content.raw", + "signature": null, + "docstring": "Raw content bytes as retrieved from the source." + }, + "source": { + "name": "source", + "kind": "attribute", + "path": "omniread.xlsx.scraper.Content.source", + "signature": null, + "docstring": "Identifier of the content origin (URL, file path, or logical name)." + }, + "content_type": { + "name": "content_type", + "kind": "attribute", + "path": "omniread.xlsx.scraper.Content.content_type", + "signature": null, + "docstring": "Optional MIME type of the content, if known." + }, + "metadata": { + "name": "metadata", + "kind": "attribute", + "path": "omniread.xlsx.scraper.Content.metadata", + "signature": null, + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." + } + } + }, + "ContentType": { + "name": "ContentType", + "kind": "class", + "path": "omniread.xlsx.scraper.ContentType", + "signature": null, + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", + "members": { + "HTML": { + "name": "HTML", + "kind": "attribute", + "path": "omniread.xlsx.scraper.ContentType.HTML", + "signature": null, + "docstring": "HTML document content." + }, + "PDF": { + "name": "PDF", + "kind": "attribute", + "path": "omniread.xlsx.scraper.ContentType.PDF", + "signature": null, + "docstring": "PDF document content." + }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.xlsx.scraper.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.xlsx.scraper.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, + "JSON": { + "name": "JSON", + "kind": "attribute", + "path": "omniread.xlsx.scraper.ContentType.JSON", + "signature": null, + "docstring": "JSON document content." + }, + "XML": { + "name": "XML", + "kind": "attribute", + "path": "omniread.xlsx.scraper.ContentType.XML", + "signature": null, + "docstring": "XML document content." + } + } + }, + "BaseXlsxClient": { + "name": "BaseXlsxClient", + "kind": "class", + "path": "omniread.xlsx.scraper.BaseXlsxClient", + "signature": null, + "docstring": "Abstract client responsible for retrieving spreadsheet bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full xlsx binary payload.\n - Raise retrieval-specific errors on failure.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.xlsx.scraper.BaseXlsxClient.fetch", + "signature": "fetch(source: Any)", + "docstring": "Fetch raw xlsx bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." + } + } + }, + "XlsxScraper": { + "name": "XlsxScraper", + "kind": "class", + "path": "omniread.xlsx.scraper.XlsxScraper", + "signature": "XlsxScraper(*, client: BaseXlsxClient)", + "docstring": "Scraper for xlsx spreadsheet documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw xlsx bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n XLSX content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.xlsx.scraper.XlsxScraper.fetch", + "signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None) -> Content", + "docstring": "Fetch an xlsx document from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw xlsx bytes, source\n identifier, XLSX content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client." + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/docs/mcp/modules/omniread.xlsx.parser.json b/docs/mcp/modules/omniread.xlsx.parser.json new file mode 100644 index 0000000..3621e93 --- /dev/null +++ b/docs/mcp/modules/omniread.xlsx.parser.json @@ -0,0 +1,106 @@ +{ + "module": "omniread.xlsx.parser", + "content": { + "path": "omniread.xlsx.parser", + "docstring": "# Summary\n\nXLSX parser implementations for OmniRead.\n\nThis module provides a concrete, generic parser for Office Open XML\nspreadsheets. It exposes workbook sheets as lists of string rows so\ndownstream consumers can interpret tabular content without depending on\nopenpyxl directly.\n\nThe parser is intentionally statement-agnostic: it performs no header\ndetection or column interpretation beyond basic cell normalization.", + "objects": { + "Content": { + "name": "Content", + "kind": "class", + "path": "omniread.xlsx.parser.Content", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", + "members": { + "raw": { + "name": "raw", + "kind": "attribute", + "path": "omniread.xlsx.parser.Content.raw", + "signature": null, + "docstring": "Raw content bytes as retrieved from the source." + }, + "source": { + "name": "source", + "kind": "attribute", + "path": "omniread.xlsx.parser.Content.source", + "signature": null, + "docstring": "Identifier of the content origin (URL, file path, or logical name)." + }, + "content_type": { + "name": "content_type", + "kind": "attribute", + "path": "omniread.xlsx.parser.Content.content_type", + "signature": null, + "docstring": "Optional MIME type of the content, if known." + }, + "metadata": { + "name": "metadata", + "kind": "attribute", + "path": "omniread.xlsx.parser.Content.metadata", + "signature": null, + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." + } + } + }, + "XlsxParserBase": { + "name": "XlsxParserBase", + "kind": "class", + "path": "omniread.xlsx.parser.XlsxParserBase", + "signature": null, + "docstring": "Base xlsx parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces xlsx content-type compatibility and provides\n the extension point for implementing concrete xlsx parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.xlsx.parser.XlsxParserBase.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser (XLSX only)." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.xlsx.parser.XlsxParserBase.parse", + "signature": "parse()", + "docstring": "Parse xlsx content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation." + } + } + }, + "XlsxParser": { + "name": "XlsxParser", + "kind": "class", + "path": "omniread.xlsx.parser.XlsxParser", + "signature": "XlsxParser(content: Content, *, data_only: bool = True, read_only: bool = True)", + "docstring": "Generic xlsx parser producing string rows from a worksheet.\n\nNotes:\n **Responsibilities:**\n\n - Lazily load the workbook owned by the parser's content.\n - Normalize cells (including dates and numeric values) into\n deterministic string representations.\n - Expose sheet discovery and row extraction helpers.\n\n **Constraints:**\n\n - Cells are rendered with ``str(value)`` after trimming; date and\n datetime values are rendered in ISO format. Consumers requiring\n locale-specific formatting must convert on their side.", + "members": { + "workbook": { + "name": "workbook", + "kind": "attribute", + "path": "omniread.xlsx.parser.XlsxParser.workbook", + "signature": null, + "docstring": "The lazily loaded workbook backing this parser's content." + }, + "sheet_names": { + "name": "sheet_names", + "kind": "attribute", + "path": "omniread.xlsx.parser.XlsxParser.sheet_names", + "signature": null, + "docstring": "Names of all worksheets contained in the workbook." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.xlsx.parser.XlsxParser.parse", + "signature": "parse() -> list[list[str]]", + "docstring": "Parse the first worksheet into normalized string rows.\n\nReturns:\n list[list[str]]:\n Rows of the default (first) worksheet." + }, + "rows": { + "name": "rows", + "kind": "function", + "path": "omniread.xlsx.parser.XlsxParser.rows", + "signature": "rows(sheet: int | str | None = None, *, skip_empty: bool = True) -> list[list[str]]", + "docstring": "Extract normalized string rows from a worksheet.\n\nArgs:\n sheet (int | str | None):\n Worksheet index or title; defaults to the first worksheet.\n skip_empty (bool):\n When True (default), rows whose cells are all blank are omitted.\n\nReturns:\n list[list[str]]:\n Normalized rows; trailing blank cells are trimmed per row.\n\nRaises:\n ValueError:\n If the requested sheet does not exist." + } + } + } + } + } +} \ No newline at end of file diff --git a/docs/mcp/modules/omniread.xlsx.parser_base.json b/docs/mcp/modules/omniread.xlsx.parser_base.json new file mode 100644 index 0000000..200e04a --- /dev/null +++ b/docs/mcp/modules/omniread.xlsx.parser_base.json @@ -0,0 +1,127 @@ +{ + "module": "omniread.xlsx.parser_base", + "content": { + "path": "omniread.xlsx.parser_base", + "docstring": "# Summary\n\nXLSX parser base implementation for OmniRead.\n\nThis module defines the **XLSX-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for Office Open\nXML spreadsheet content.", + "objects": { + "ContentType": { + "name": "ContentType", + "kind": "class", + "path": "omniread.xlsx.parser_base.ContentType", + "signature": null, + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", + "members": { + "HTML": { + "name": "HTML", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.ContentType.HTML", + "signature": null, + "docstring": "HTML document content." + }, + "PDF": { + "name": "PDF", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.ContentType.PDF", + "signature": null, + "docstring": "PDF document content." + }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, + "JSON": { + "name": "JSON", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.ContentType.JSON", + "signature": null, + "docstring": "JSON document content." + }, + "XML": { + "name": "XML", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.ContentType.XML", + "signature": null, + "docstring": "XML document content." + } + } + }, + "BaseParser": { + "name": "BaseParser", + "kind": "class", + "path": "omniread.xlsx.parser_base.BaseParser", + "signature": "BaseParser(content: Content)", + "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.BaseParser.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." + }, + "content": { + "name": "content", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.BaseParser.content", + "signature": null, + "docstring": null + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.xlsx.parser_base.BaseParser.parse", + "signature": "parse()", + "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." + }, + "supports": { + "name": "supports", + "kind": "function", + "path": "omniread.xlsx.parser_base.BaseParser.supports", + "signature": "supports()", + "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." + } + } + }, + "T": { + "name": "T", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.T", + "signature": null, + "docstring": null + }, + "XlsxParserBase": { + "name": "XlsxParserBase", + "kind": "class", + "path": "omniread.xlsx.parser_base.XlsxParserBase", + "signature": null, + "docstring": "Base xlsx parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces xlsx content-type compatibility and provides\n the extension point for implementing concrete xlsx parsing\n strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", + "members": { + "supported_types": { + "name": "supported_types", + "kind": "attribute", + "path": "omniread.xlsx.parser_base.XlsxParserBase.supported_types", + "signature": null, + "docstring": "Set of content types supported by this parser (XLSX only)." + }, + "parse": { + "name": "parse", + "kind": "function", + "path": "omniread.xlsx.parser_base.XlsxParserBase.parse", + "signature": "parse() -> T", + "docstring": "Parse xlsx content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation." + } + } + } + } + } +} \ No newline at end of file diff --git a/docs/mcp/modules/omniread.xlsx.scraper.json b/docs/mcp/modules/omniread.xlsx.scraper.json new file mode 100644 index 0000000..7e5cdac --- /dev/null +++ b/docs/mcp/modules/omniread.xlsx.scraper.json @@ -0,0 +1,129 @@ +{ + "module": "omniread.xlsx.scraper", + "content": { + "path": "omniread.xlsx.scraper", + "docstring": "# Summary\n\nXLSX scraper for OmniRead.\n\nThis module defines the scraper responsible for acquiring raw Office Open\nXML spreadsheet content from a backing store via a configured client.\n\nThe scraper does not interpret or parse the acquired bytes; it wraps them in\nthe canonical `Content` model.", + "objects": { + "Content": { + "name": "Content", + "kind": "class", + "path": "omniread.xlsx.scraper.Content", + "signature": "Content(raw: bytes, source: str, content_type: ContentType | None = ..., metadata: Mapping[str, Any] | None = ...)", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", + "members": { + "raw": { + "name": "raw", + "kind": "attribute", + "path": "omniread.xlsx.scraper.Content.raw", + "signature": null, + "docstring": "Raw content bytes as retrieved from the source." + }, + "source": { + "name": "source", + "kind": "attribute", + "path": "omniread.xlsx.scraper.Content.source", + "signature": null, + "docstring": "Identifier of the content origin (URL, file path, or logical name)." + }, + "content_type": { + "name": "content_type", + "kind": "attribute", + "path": "omniread.xlsx.scraper.Content.content_type", + "signature": null, + "docstring": "Optional MIME type of the content, if known." + }, + "metadata": { + "name": "metadata", + "kind": "attribute", + "path": "omniread.xlsx.scraper.Content.metadata", + "signature": null, + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." + } + } + }, + "ContentType": { + "name": "ContentType", + "kind": "class", + "path": "omniread.xlsx.scraper.ContentType", + "signature": null, + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", + "members": { + "HTML": { + "name": "HTML", + "kind": "attribute", + "path": "omniread.xlsx.scraper.ContentType.HTML", + "signature": null, + "docstring": "HTML document content." + }, + "PDF": { + "name": "PDF", + "kind": "attribute", + "path": "omniread.xlsx.scraper.ContentType.PDF", + "signature": null, + "docstring": "PDF document content." + }, + "XLSX": { + "name": "XLSX", + "kind": "attribute", + "path": "omniread.xlsx.scraper.ContentType.XLSX", + "signature": null, + "docstring": "Office Open XML spreadsheet (xlsx/xlsm) content." + }, + "CSV": { + "name": "CSV", + "kind": "attribute", + "path": "omniread.xlsx.scraper.ContentType.CSV", + "signature": null, + "docstring": "Comma-separated-value document content." + }, + "JSON": { + "name": "JSON", + "kind": "attribute", + "path": "omniread.xlsx.scraper.ContentType.JSON", + "signature": null, + "docstring": "JSON document content." + }, + "XML": { + "name": "XML", + "kind": "attribute", + "path": "omniread.xlsx.scraper.ContentType.XML", + "signature": null, + "docstring": "XML document content." + } + } + }, + "BaseXlsxClient": { + "name": "BaseXlsxClient", + "kind": "class", + "path": "omniread.xlsx.scraper.BaseXlsxClient", + "signature": null, + "docstring": "Abstract client responsible for retrieving spreadsheet bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full xlsx binary payload.\n - Raise retrieval-specific errors on failure.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.xlsx.scraper.BaseXlsxClient.fetch", + "signature": "fetch(source: Any)", + "docstring": "Fetch raw xlsx bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw xlsx bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." + } + } + }, + "XlsxScraper": { + "name": "XlsxScraper", + "kind": "class", + "path": "omniread.xlsx.scraper.XlsxScraper", + "signature": "XlsxScraper(*, client: BaseXlsxClient)", + "docstring": "Scraper for xlsx spreadsheet documents.\n\nNotes:\n **Responsibilities:**\n\n - Fetch raw xlsx bytes via the configured client.\n - Wrap the payload in a canonical `Content` instance with the\n XLSX content type and source identifier.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", + "members": { + "fetch": { + "name": "fetch", + "kind": "function", + "path": "omniread.xlsx.scraper.XlsxScraper.fetch", + "signature": "fetch(source: Any, *, metadata: Mapping[str, Any] | None = None) -> Content", + "docstring": "Fetch an xlsx document from the given source.\n\nArgs:\n source (Any):\n Identifier of the spreadsheet source as understood by the\n configured client.\n metadata (Mapping[str, Any] | None, optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw xlsx bytes, source\n identifier, XLSX content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the client." + } + } + } + } + } +} \ No newline at end of file diff --git a/docs/mcp/nav.json b/docs/mcp/nav.json new file mode 100644 index 0000000..1cc5251 --- /dev/null +++ b/docs/mcp/nav.json @@ -0,0 +1,90 @@ +[ + { + "module": "omniread", + "resource": "docs://modules/omniread" + }, + { + "module": "omniread.core", + "resource": "docs://modules/omniread.core" + }, + { + "module": "omniread.core.content", + "resource": "docs://modules/omniread.core.content" + }, + { + "module": "omniread.core.parser", + "resource": "docs://modules/omniread.core.parser" + }, + { + "module": "omniread.core.scraper", + "resource": "docs://modules/omniread.core.scraper" + }, + { + "module": "omniread.csv", + "resource": "docs://modules/omniread.csv" + }, + { + "module": "omniread.csv.client", + "resource": "docs://modules/omniread.csv.client" + }, + { + "module": "omniread.csv.parser", + "resource": "docs://modules/omniread.csv.parser" + }, + { + "module": "omniread.csv.parser_base", + "resource": "docs://modules/omniread.csv.parser_base" + }, + { + "module": "omniread.csv.scraper", + "resource": "docs://modules/omniread.csv.scraper" + }, + { + "module": "omniread.html", + "resource": "docs://modules/omniread.html" + }, + { + "module": "omniread.html.parser", + "resource": "docs://modules/omniread.html.parser" + }, + { + "module": "omniread.html.scraper", + "resource": "docs://modules/omniread.html.scraper" + }, + { + "module": "omniread.pdf", + "resource": "docs://modules/omniread.pdf" + }, + { + "module": "omniread.pdf.client", + "resource": "docs://modules/omniread.pdf.client" + }, + { + "module": "omniread.pdf.parser", + "resource": "docs://modules/omniread.pdf.parser" + }, + { + "module": "omniread.pdf.scraper", + "resource": "docs://modules/omniread.pdf.scraper" + }, + { + "module": "omniread.xlsx", + "resource": "docs://modules/omniread.xlsx" + }, + { + "module": "omniread.xlsx.client", + "resource": "docs://modules/omniread.xlsx.client" + }, + { + "module": "omniread.xlsx.parser", + "resource": "docs://modules/omniread.xlsx.parser" + }, + { + "module": "omniread.xlsx.parser_base", + "resource": "docs://modules/omniread.xlsx.parser_base" + }, + { + "module": "omniread.xlsx.scraper", + "resource": "docs://modules/omniread.xlsx.scraper" + } +] \ No newline at end of file diff --git a/mkdocs.yml b/docs/mkdocs.lib.yml similarity index 79% rename from mkdocs.yml rename to docs/mkdocs.lib.yml index 4ab42e7..a89652d 100644 --- a/mkdocs.yml +++ b/docs/mkdocs.lib.yml @@ -21,33 +21,15 @@ theme: - search.highlight - search.share - search.suggest -plugins: -- search -- mkdocstrings: - handlers: - python: - paths: - - . - options: - docstring_style: google - show_source: false - show_signature_annotations: true - separate_signature: true - merge_init_into_class: true - inherited_members: true - annotations_path: brief - show_root_heading: true - group_by_category: true - show_category_heading: true - show_object_full_path: false - show_symbol_type_heading: true + icon: + logo: material/book-open-page-variant + repo: fontawesome/brands/github markdown_extensions: - pymdownx.superfences - pymdownx.inlinehilite - pymdownx.snippets - admonition - pymdownx.details -- pymdownx.superfences - pymdownx.highlight: linenums: true anchor_linenums: true @@ -62,7 +44,31 @@ markdown_extensions: - pymdownx.caret - pymdownx.tilde - pymdownx.mark +extra_css: +- https://unpkg.com/dracula-prism/dist/css/dracula-prism.css +plugins: +- search +- mkdocstrings: + handlers: + python: + paths: + - .. + options: + docstring_style: google + show_source: false + show_signature_annotations: true + separate_signature: true + merge_init_into_class: true + inherited_members: true + annotations_path: brief + show_root_heading: true + group_by_category: true + show_category_heading: true + show_object_full_path: false + show_symbol_type_heading: true site_name: omniread +docs_dir: lib +site_dir: ../site/lib nav: - Home: index.md - Core API: @@ -79,3 +85,15 @@ nav: - pdf/client.md - pdf/parser.md - pdf/scraper.md +- CSV Handling: + - csv/index.md + - csv/client.md + - csv/parser_base.md + - csv/parser.md + - csv/scraper.md +- XLSX Handling: + - xlsx/index.md + - xlsx/client.md + - xlsx/parser_base.md + - xlsx/parser.md + - xlsx/scraper.md diff --git a/docs/mkdocs.wiki.yml b/docs/mkdocs.wiki.yml new file mode 100644 index 0000000..03e9e71 --- /dev/null +++ b/docs/mkdocs.wiki.yml @@ -0,0 +1,69 @@ +site_name: OmniRead Documentation +docs_dir: wiki +site_dir: ../site/wiki + +nav: +- Home: index.md +- Overview: 01_overview.md +- How to Use: 02_how_to_use.md +- Extending OmniRead: 03_extending.md +- Development: 04_development.md + +theme: + name: material + palette: + scheme: slate + primary: blue grey + accent: teal + font: + text: Roboto + code: JetBrains Mono + features: + - navigation.sections + - navigation.expand + - navigation.top + - navigation.instant + - navigation.tracking + - navigation.indexes + - content.code.copy + - content.code.annotate + - content.tabs.link + - content.action.edit + - search.highlight + - search.share + - search.suggest + - navigation.tabs + - toc.integrate + - header.autohide + - announce.dismiss + - footer.social + - content.code.select + - content.code.line_numbers + - content.tooltips + icon: + logo: material/book-open-page-variant + repo: fontawesome/brands/github +markdown_extensions: +- pymdownx.superfences +- pymdownx.inlinehilite +- pymdownx.snippets +- admonition +- pymdownx.details +- pymdownx.highlight: + linenums: true + anchor_linenums: true + line_spans: __span + pygments_lang_class: true +- pymdownx.tabbed: + alternate_style: true +- pymdownx.tasklist: + custom_checkbox: true +- tables +- footnotes +- pymdownx.caret +- pymdownx.tilde +- pymdownx.mark +extra_css: +- https://unpkg.com/dracula-prism/dist/css/dracula-prism.css +plugins: +- search \ No newline at end of file diff --git a/docs/wiki/01_overview.md b/docs/wiki/01_overview.md new file mode 100644 index 0000000..b7d846f --- /dev/null +++ b/docs/wiki/01_overview.md @@ -0,0 +1,86 @@ +# 🧱 Overview + +OmniRead is designed as a **decoupled content engine** with three distinct +layers. Understanding them is the key to using and extending the library. + +--- + +## 🏗️ Architecture + +```text + ┌─────────────────────────────┐ + │ Source (URL, file, storage) │ + └────────────┬────────────────┘ + │ + ┌─────────▼──────────┐ + │ Scraper / Client │ fetches raw bytes + └─────────┬──────────┘ + │ returns + ┌─────────▼──────────┐ + │ Content │ raw + source + type + └─────────┬──────────┘ + │ + ┌─────────▼──────────┐ + │ Parser │ parse() → structured T + └────────────────────┘ +``` + +1. **Scraper** (`BaseScraper`) — fetches raw bytes from a source (HTTP URL, + filesystem path, object storage). Returns a `Content` instance. + Scrapers never interpret content. +2. **Content** (`Content`) — the canonical exchange model: raw bytes, + source identifier, and optional `ContentType` enum. +3. **Parser** (`BaseParser[T]`) — receives a `Content` and returns a + structured result of type `T` via `parse()`. + +--- + +## 📦 The `Content` model + +Defined in `omniread.core.content`: + +```python +from dataclasses import dataclass +from omniread import Content, ContentType + +@dataclass(slots=True) +class Content: + raw: bytes + source: str + content_type: ContentType | None = None +``` + +- `raw` — the raw bytes exactly as retrieved. +- `source` — URL, file path, or logical name identifying the origin. +- `content_type` — optional `ContentType` enum value. + +--- + +## 🎭 The `ContentType` enum + +| Value | MIME | Used by | +|---|---|---| +| `HTML` | `text/html` | HTMLScraper | +| `PDF` | `application/pdf` | PDFScraper | +| `XLSX` | `application/vnd.openxmlformats-...` | XlsxScraper | +| `CSV` | `text/csv` | CsvScraper | +| `JSON` | `application/json` | — | +| `XML` | `application/xml` | — | + +--- + +## 🧩 Format modules at a glance + +| Module | Scraper | Parser | Client | Notes | +|---|---|---|---|---| +| `omniread.html` | `HTMLScraper` | `HTMLParser` | — | httpx + BeautifulSoup | +| `omniread.pdf` | `PDFScraper` | `PDFParser` | `FileSystemPDFClient` | explicit client layer | +| `omniread.csv` | `CsvScraper` | `CsvParser` | `FileSystemCsvClient` | stdlib csv module | +| `omniread.xlsx` | `XlsxScraper` | `XlsxParser` | `FileSystemXlsxClient` | openpyxl-backed | + +--- + +## 📚 Read Next + +- [How to Use](02_how_to_use.md) — working examples per format. +- [Extending OmniRead](03_extending.md) — subclassing scrapers and parsers. \ No newline at end of file diff --git a/docs/wiki/02_how_to_use.md b/docs/wiki/02_how_to_use.md new file mode 100644 index 0000000..1fd1974 --- /dev/null +++ b/docs/wiki/02_how_to_use.md @@ -0,0 +1,109 @@ +# 🖥️ How to Use + +Every format follows the same pipeline: **scrape → `Content` → parse**. +This page shows each supported format with working patterns. + +--- + +## 🌐 HTML + +```python +from omniread import HTMLScraper, HTMLParser + +class TitleParser(HTMLParser[str]): + def parse(self) -> str: + return self._soup.title.string + +scraper = HTMLScraper() # httpx under the hood +content = scraper.fetch("https://example.com") + +title = TitleParser(content).parse() +``` + +`HTMLScraper` accepts an optional `client` (an `httpx.Client`) for transport +control — the test suite wires one to a mock transport. + +--- + +## 📕 PDF + +PDFs need a **client** to supply raw bytes before parsing: + +```python +from pathlib import Path +from omniread import FileSystemPDFClient, PDFScraper, PDFParser + +class TextPDFParser(PDFParser[str]): + def parse(self) -> str: + # implement your extraction logic + return self.content.raw # bytes, decode as needed + +client = FileSystemPDFClient() +scraper = PDFScraper(client=client) +content = scraper.fetch(Path("document.pdf")) + +result = TextPDFParser(content).parse() +``` + +`PDFParser` subclasses receive `self.content` and implement `parse()`. + +--- + +## 📊 CSV + +```python +from omniread import FileSystemCsvClient, CsvScraper, CsvParser + +scraper = CsvScraper(client=FileSystemCsvClient()) +content = scraper.fetch("data.csv") + +parser = CsvParser(content) +for row in parser.rows(): + print(row) +``` + +`CsvParser.rows()` yields string rows trimmed of empties by default +(`skip_empty=True`). + +--- + +## 📑 XLSX + +```python +from omniread import FileSystemXlsxClient, XlsxScraper, XlsxParser + +scraper = XlsxScraper(client=FileSystemXlsxClient()) +content = scraper.fetch("statement.xlsx") + +parser = XlsxParser(content) +print(parser.sheet_names) # e.g. ["Statement"] +rows = parser.rows(sheet="Statement") # by name or index +all_rows = parser.parse() # alias for rows() +``` + +Key behaviors: + +- `rows(skip_empty=False)` keeps blank rows (off by default). +- Cells render as trimmed strings; date cells convert to ISO format + (`2026-06-01T00:00:00`). +- `rows(sheet="Missing")` raises for an unknown sheet. + +--- + +## 🔀 End-to-end flow + +```python +content = scraper.fetch(source) # 1. acquire → Content +assert isinstance(content.raw, bytes) +assert content.content_type is not None + +parser = MyParser(content) # 2. interpret → T +result = parser.parse() +``` + +--- + +## 📚 Read Next + +- [Overview](01_overview.md) — the full architecture. +- [Extending OmniRead](03_extending.md) — custom clients and parsers. \ No newline at end of file diff --git a/docs/wiki/03_extending.md b/docs/wiki/03_extending.md new file mode 100644 index 0000000..da2ff35 --- /dev/null +++ b/docs/wiki/03_extending.md @@ -0,0 +1,104 @@ +# 🧩 Extending OmniRead + +OmniRead is meant to be extended by subclassing. All public extension points +are generic over their result type, so your parser returns exactly the shape +you need. + +--- + +## 🧬 Custom parsers + +Subclass `BaseParser[T]` (or a format parser) and implement `parse()`: + +```python +from pydantic import BaseModel +from omniread import HTMLParser + +class Page(BaseModel): + title: str + content: str | None + +class PageParser(HTMLParser[Page]): + def parse(self) -> Page: + soup = self._soup + div = soup.find("div", id="content") + return Page( + title=soup.title.string, + content=div.get_text() if div else None, + ) +``` + +The parsed page is validated by Pydantic on construction — no manual +assertions required. + +--- + +## 🧬 Custom PDF parsers + +PDF binary layout is format-specific, so parsers return your own model: + +```python +from typing import Literal +from pydantic import BaseModel +from omniread import PDFParser + +class ParsedPDF(BaseModel): + size_bytes: int + magic: Literal[b"%PDF"] + +class SimplePDFParser(PDFParser[ParsedPDF]): + def parse(self) -> ParsedPDF: + if not self.content.raw.startswith(b"%PDF"): + raise ValueError("Not a valid PDF") + return ParsedPDF(size_bytes=len(self.content.raw), magic=b"%PDF") +``` + +--- + +## 🧬 Custom clients + +Clients supply raw bytes to a scraper. For PDFs, subclass +`BasePDFClient` (or `FileSystemPDFClient`) and implement +`fetch(source) -> bytes`: + +```python +from omniread.pdf.client import BasePDFClient + +class MockPDFClient(BasePDFClient): + def fetch(self, source): + return b"%PDF ..." # bytes for the logical identifier +``` + +The same pattern applies to `BaseCsvClient` and `BaseXlsxClient`. + +--- + +## 🚀 Custom scrapers + +Festch something that a built-in scraper does not cover by extending +`BaseScraper`: + +```python +from omniread import BaseScraper, Content, ContentType + +class StorageScraper(BaseScraper): + def fetch(self, source, *, metadata=None): + raw = my_object_storage.download(source) # your I/O + return Content(raw=raw, source=source, content_type=ContentType.JSON) +``` + +--- + +## ✅ Extension checklist + +1. Keep **scraper** and **parser** separate — never mix I/O into `parse()`. +2. Return `Content` from any scraper/client so downstream stays uniform. +3. Return a *typed* result from your parser (Pydantic model, dataclass, str). +4. Test your custom layers with a mock client, not a live network. + +--- + +## 📚 Read Next + +- [How to Use](02_how_to_use.md) — built-in example flows. +- [Development](04_development.md) — running tests and docs. \ No newline at end of file diff --git a/docs/wiki/04_development.md b/docs/wiki/04_development.md new file mode 100644 index 0000000..69e4f8c --- /dev/null +++ b/docs/wiki/04_development.md @@ -0,0 +1,87 @@ +# 🛠️ Development + +Working on `omniread` itself. + +--- + +## 📂 Repository layout + +| Path | Purpose | +|---|---| +| `omniread/` | The library package (core + html, pdf, csv, xlsx modules) | +| `omniread/*.pyi` | Type stubs kept in sync with implementations | +| `tests/` | End-to-end and unit tests against mock transports/clients | +| `covers` | — | +| `docs/lib/` | Generated library reference (docforge, flat layout) | +| `docs/mcp/` | Machine-readable bundle served by the MCP server | +| `docs/wiki/` | This hand-written wiki | + +--- + +## 🔧 Setup + +```bash +python -m venv .venv +.venv/Scripts/pip install -e ".[dev]" +``` + +--- + +## 🧪 Tests + +Run the suite (offline; a mock httpx transport and mock PDF client are used): + +```bash +.venv/Scripts/pytest +``` + +Coverage spans end-to-end scrape → parse flows for HTML, PDF, and XLSX, +plus client validation and CSV/XLSX parsing edge cases. + +--- + +## ✅ Quality gates + +The CI quality gate runs, matching the Drone pipeline: + +```bash +.venv/Scripts/black --check . +.venv/Scripts/ruff check . +.venv/Scripts/mypy +.venv/Scripts/pytest +``` + +--- + +## 📝 Building documentation (docforge) + +The site is generated by [`docforge`](https://git.aetoskia.com/aetos/doc-forge) +and served per kind under `site/{kind}`: + +```bash +doc-forge build \ + --mkdocs --mcp \ + --module-is-source --module omniread \ + --site-name "OmniRead" +``` + +- `--module-is-source` renders the flat `docs/lib/` layout (no nesting under + `omniread/`), matching `docforge.nav.yml` and `docs/mkdocs.lib.yml`. +- `--mcp` regenerates the structured bundle in `docs/mcp/`. +- `--wiki` builds this wiki. +- `gen_api.py` (if present) regenerates the API docs. + +Preview locally: + +```bash +doc-forge serve --lib +doc-forge serve --wiki +doc-forge serve --mcp +``` + +--- + +## 📚 Read Next + +- [Extending OmniRead](03_extending.md) — custom parsers, scrapers, clients. +- [Overview](01_overview.md) — the core architecture. \ No newline at end of file diff --git a/docs/wiki/index.md b/docs/wiki/index.md new file mode 100644 index 0000000..7256d4b --- /dev/null +++ b/docs/wiki/index.md @@ -0,0 +1,64 @@ +# 📄 OmniRead — Format-Agnostic Content Acquisition and Parsing + +`OmniRead` is a composable content ingestion framework with pluggable scrapers +and parsers for HTML, PDF, CSV, and XLSX. It cleanly separates fetching +(I/O) from interpretation (parsing) through a canonical `Content` exchange +model, giving each format its own scraper and parser while sharing one +uniform pipeline. + +> **Doc model:** this wiki is written for humans — how‑to guides, examples, +> and extension recipes. The authoritative API contracts live in the code +> (GSDFC docstrings) and the machine‑readable bundle under `docs/mcp/`. + +--- + +## 🚀 Key Features + +* 🧱 **Content/Scraper/Parser architecture** — fetch and interpret in + independent, testable layers +* 📄 **Canonical `Content` model** — raw bytes + metadata passed between all + layers via one dataclass +* 🌐 **HTML** — HTTP-based scraper with httpx; DOM parsing via BeautifulSoup +* 📕 **PDF** — client + scraper for raw PDF bytes; user-defined parsers +* 📊 **CSV** — standard-library CSV parser over `rows()` iterator +* 📑 **XLSX** — openpyxl-backed spreadsheet parser with sheet selection, + empty-row control, and ISO date conversion + +--- + +## ⚡ Quick Start + +```python +from omniread import HTMLScraper, HTMLParser + +class TitleParser(HTMLParser[str]): + def parse(self) -> str: + return self._soup.title.string + +content = HTMLScraper().fetch("https://example.com") +title = TitleParser(content).parse() +assert title == "Example Domain" +``` + +--- + +## 📁 Documentation Structure + +| Section | What you'll find | +|---|---| +| [Overview](01_overview.md) | The three-layer architecture and the `Content` model | +| [How to Use](02_how_to_use.md) | Ready-to-run examples for HTML, PDF, CSV, and XLSX | +| [Extending OmniRead](03_extending.md) | Subclassing scrapers, parsers, and clients | +| [Development](04_development.md) | Setup, tests, and regenerating docs | + +--- + +## 🔗 Related Resources + +* **Source Code:** [Gitea Repository](https://git.aetoskia.com/aetos/omniread) +* **Internal PyPI:** [pip.aetoskia.com/simple/omniread](https://pip.aetoskia.com/simple/omniread) +* **CI:** Builds and publishes tagged releases, gated on black / ruff / mypy / pytest. + +--- + +© Aetoskia Internal — `omniread` 0.0.1 \ No newline at end of file diff --git a/mcp_docs/modules/omniread.json b/mcp_docs/modules/omniread.json deleted file mode 100644 index 8b122a6..0000000 --- a/mcp_docs/modules/omniread.json +++ /dev/null @@ -1,1639 +0,0 @@ -{ - "module": "omniread", - "content": { - "path": "omniread", - "docstring": "# Summary\n\n`OmniRead` — format-agnostic content acquisition and parsing framework.\n\n`OmniRead` provides a **cleanly layered architecture** for fetching, parsing,\nand normalizing content from heterogeneous sources such as HTML documents\nand PDF files.\n\nThe library is structured around three core concepts:\n\n1. **`Content`**: A canonical, format-agnostic container representing raw content\n bytes and minimal contextual metadata.\n2. **`Scrapers`**: Components responsible for *acquiring* raw content from a\n source (HTTP, filesystem, object storage, etc.). `Scrapers` never interpret\n content.\n3. **`Parsers`**: Components responsible for *interpreting* acquired content and\n converting it into structured, typed representations.\n\n`OmniRead` deliberately separates these responsibilities to ensure:\n\n- Clear boundaries between IO and interpretation.\n- Replaceable implementations per format.\n- Predictable, testable behavior.\n\n# Installation\n\nInstall `OmniRead` using pip:\n\n```bash\npip install omniread\n```\n\nInstall OmniRead using Poetry:\n```bash\npoetry add omniread\n```\n\n---\n\n## Quick start\n\nExample:\n HTML example:\n ```python\n from omniread import HTMLScraper, HTMLParser\n\n scraper = HTMLScraper()\n content = scraper.fetch(\"https://example.com\")\n\n class TitleParser(HTMLParser[str]):\n def parse(self) -> str:\n return self._soup.title.string\n\n parser = TitleParser(content)\n title = parser.parse()\n ```\n\n PDF example:\n ```python\n from omniread import FileSystemPDFClient, PDFScraper, PDFParser\n from pathlib import Path\n\n client = FileSystemPDFClient()\n scraper = PDFScraper(client=client)\n content = scraper.fetch(Path(\"document.pdf\"))\n\n class TextPDFParser(PDFParser[str]):\n def parse(self) -> str:\n # implement PDF text extraction\n ...\n\n parser = TextPDFParser(content)\n result = parser.parse()\n ```\n\n---\n\n# Public API\n\nThis module re-exports the **recommended public entry points** of OmniRead.\nConsumers are encouraged to import from this namespace rather than from\nformat-specific submodules directly, unless advanced customization is\nrequired.\n\n- `Content`: Canonical content model.\n- `ContentType`: Supported media types.\n- `HTMLScraper`: HTTP-based HTML acquisition.\n- `HTMLParser`: Base parser for HTML DOM interpretation.\n- `FileSystemPDFClient`: Local filesystem PDF access.\n- `PDFScraper`: PDF-specific content acquisition.\n- `PDFParser`: Base parser for PDF binary interpretation.\n\n---\n\n# Core Philosophy\n\n`OmniRead` is designed as a **decoupled content engine**:\n\n1. **Separation of Concerns**: Scrapers *fetch*, Parsers *interpret*. Neither\n knows about the other.\n2. **Normalized Exchange**: All components communicate via the `Content` model,\n ensuring a consistent contract.\n3. **Format Agnosticism**: The core logic is independent of whether the input\n is HTML, PDF, or JSON.\n\n---", - "objects": { - "Content": { - "name": "Content", - "kind": "class", - "path": "omniread.Content", - "signature": "", - "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", - "members": { - "raw": { - "name": "raw", - "kind": "attribute", - "path": "omniread.Content.raw", - "signature": "", - "docstring": "Raw content bytes as retrieved from the source." - }, - "source": { - "name": "source", - "kind": "attribute", - "path": "omniread.Content.source", - "signature": "", - "docstring": "Identifier of the content origin (URL, file path, or logical name)." - }, - "content_type": { - "name": "content_type", - "kind": "attribute", - "path": "omniread.Content.content_type", - "signature": "", - "docstring": "Optional MIME type of the content, if known." - }, - "metadata": { - "name": "metadata", - "kind": "attribute", - "path": "omniread.Content.metadata", - "signature": "", - "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." - } - } - }, - "ContentType": { - "name": "ContentType", - "kind": "class", - "path": "omniread.ContentType", - "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", - "members": { - "HTML": { - "name": "HTML", - "kind": "attribute", - "path": "omniread.ContentType.HTML", - "signature": "", - "docstring": "HTML document content." - }, - "PDF": { - "name": "PDF", - "kind": "attribute", - "path": "omniread.ContentType.PDF", - "signature": "", - "docstring": "PDF document content." - }, - "JSON": { - "name": "JSON", - "kind": "attribute", - "path": "omniread.ContentType.JSON", - "signature": "", - "docstring": "JSON document content." - }, - "XML": { - "name": "XML", - "kind": "attribute", - "path": "omniread.ContentType.XML", - "signature": "", - "docstring": "XML document content." - } - } - }, - "HTMLScraper": { - "name": "HTMLScraper", - "kind": "class", - "path": "omniread.HTMLScraper", - "signature": "", - "docstring": "Base HTML scraper using `httpx`.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns\n them as raw content wrapped in a `Content` object.\n - Fetches raw bytes and metadata only.\n - The scraper uses `httpx.Client` for HTTP requests, enforces an\n HTML content type, and preserves HTTP response metadata.\n\n **Constraints:**\n\n - The scraper does not: Parse HTML, perform retries or backoff,\n handle non-HTML responses.", - "members": { - "content_type": { - "name": "content_type", - "kind": "attribute", - "path": "omniread.HTMLScraper.content_type", - "signature": "", - "docstring": null - }, - "validate_content_type": { - "name": "validate_content_type", - "kind": "function", - "path": "omniread.HTMLScraper.validate_content_type", - "signature": "", - "docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content." - }, - "fetch": { - "name": "fetch", - "kind": "function", - "path": "omniread.HTMLScraper.fetch", - "signature": "", - "docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML." - } - } - }, - "HTMLParser": { - "name": "HTMLParser", - "kind": "class", - "path": "omniread.HTMLParser", - "signature": "", - "docstring": "Base HTML parser.\n\nNotes:\n **Responsibilities:**\n\n - This class extends the core `BaseParser` with HTML-specific behavior,\n including DOM parsing via BeautifulSoup and reusable extraction helpers.\n - Provides reusable helpers for HTML extraction. Concrete parsers must\n explicitly define the return type.\n\n **Guarantees:**\n\n - Accepts only HTML content.\n - Owns a parsed BeautifulSoup DOM tree.\n - Provides pure helper utilities for common HTML structures.\n\n **Constraints:**\n\n - Concrete subclasses must define the output type `T` and implement\n the `parse()` method.", - "members": { - "supported_types": { - "name": "supported_types", - "kind": "attribute", - "path": "omniread.HTMLParser.supported_types", - "signature": "", - "docstring": "Set of content types supported by this parser (HTML only)." - }, - "parse": { - "name": "parse", - "kind": "function", - "path": "omniread.HTMLParser.parse", - "signature": "", - "docstring": "Fully parse the HTML content into structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the HTML DOM and return a\n deterministic, structured output." - }, - "parse_div": { - "name": "parse_div", - "kind": "function", - "path": "omniread.HTMLParser.parse_div", - "signature": "", - "docstring": "Extract normalized text from a `
` element.\n\nArgs:\n div (Tag):\n BeautifulSoup tag representing a `
`.\n\nReturns:\n list[list[str]]:\n A list of rows, where each row is a list of cell text values." - }, - "parse_meta": { - "name": "parse_meta", - "kind": "function", - "path": "omniread.HTMLParser.parse_meta", - "signature": "", - "docstring": "Extract high-level metadata from the HTML document.\n\nReturns:\n dict[str, Any]:\n Dictionary containing extracted metadata.\n\nNotes:\n **Responsibilities:**\n\n - Extract high-level metadata from the HTML document.\n - This includes: Document title, `` tag name/property to\n content mappings." - } - } - }, - "FileSystemPDFClient": { - "name": "FileSystemPDFClient", - "kind": "class", - "path": "omniread.FileSystemPDFClient", - "signature": "", - "docstring": "PDF client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads PDF files directly from the disk and returns\n their raw binary contents.", - "members": { - "fetch": { - "name": "fetch", - "kind": "function", - "path": "omniread.FileSystemPDFClient.fetch", - "signature": "", - "docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the PDF file.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." - } - } - }, - "PDFScraper": { - "name": "PDFScraper", - "kind": "class", - "path": "omniread.PDFScraper", - "signature": "", - "docstring": "Scraper for PDF sources.\n\nNotes:\n **Responsibilities:**\n\n - Delegates byte retrieval to a PDF client and normalizes output\n into `Content`.\n - Preserves caller-provided metadata.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", - "members": { - "fetch": { - "name": "fetch", - "kind": "function", - "path": "omniread.PDFScraper.fetch", - "signature": "", - "docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client." - } - } - }, - "PDFParser": { - "name": "PDFParser", - "kind": "class", - "path": "omniread.PDFParser", - "signature": "", - "docstring": "Base PDF parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces PDF content-type compatibility and provides\n the extension point for implementing concrete PDF parsing strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", - "members": { - "supported_types": { - "name": "supported_types", - "kind": "attribute", - "path": "omniread.PDFParser.supported_types", - "signature": "", - "docstring": "Set of content types supported by this parser (PDF only)." - }, - "parse": { - "name": "parse", - "kind": "function", - "path": "omniread.PDFParser.parse", - "signature": "", - "docstring": "Parse PDF content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the PDF binary payload and\n return a deterministic, structured output." - } - } - }, - "core": { - "name": "core", - "kind": "module", - "path": "omniread.core", - "signature": null, - "docstring": "# Summary\n\nCore domain contracts for OmniRead.\n\nThis package defines the **format-agnostic domain layer** of OmniRead.\nIt exposes canonical content models and abstract interfaces that are\nimplemented by format-specific modules (HTML, PDF, etc.).\n\nPublic exports from this package are considered **stable contracts** and\nare safe for downstream consumers to depend on.\n\nSubmodules:\n\n- `content`: Canonical content models and enums.\n- `parser`: Abstract parsing contracts.\n- `scraper`: Abstract scraping contracts.\n\nFormat-specific behavior must not be introduced at this layer.\n\n---\n\n# Public API\n\n- `Content`\n- `ContentType`\n\n---", - "members": { - "Content": { - "name": "Content", - "kind": "class", - "path": "omniread.core.Content", - "signature": "", - "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", - "members": { - "raw": { - "name": "raw", - "kind": "attribute", - "path": "omniread.core.Content.raw", - "signature": "", - "docstring": "Raw content bytes as retrieved from the source." - }, - "source": { - "name": "source", - "kind": "attribute", - "path": "omniread.core.Content.source", - "signature": "", - "docstring": "Identifier of the content origin (URL, file path, or logical name)." - }, - "content_type": { - "name": "content_type", - "kind": "attribute", - "path": "omniread.core.Content.content_type", - "signature": "", - "docstring": "Optional MIME type of the content, if known." - }, - "metadata": { - "name": "metadata", - "kind": "attribute", - "path": "omniread.core.Content.metadata", - "signature": "", - "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." - } - } - }, - "ContentType": { - "name": "ContentType", - "kind": "class", - "path": "omniread.core.ContentType", - "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", - "members": { - "HTML": { - "name": "HTML", - "kind": "attribute", - "path": "omniread.core.ContentType.HTML", - "signature": "", - "docstring": "HTML document content." - }, - "PDF": { - "name": "PDF", - "kind": "attribute", - "path": "omniread.core.ContentType.PDF", - "signature": "", - "docstring": "PDF document content." - }, - "JSON": { - "name": "JSON", - "kind": "attribute", - "path": "omniread.core.ContentType.JSON", - "signature": "", - "docstring": "JSON document content." - }, - "XML": { - "name": "XML", - "kind": "attribute", - "path": "omniread.core.ContentType.XML", - "signature": "", - "docstring": "XML document content." - } - } - }, - "BaseParser": { - "name": "BaseParser", - "kind": "class", - "path": "omniread.core.BaseParser", - "signature": "", - "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", - "members": { - "supported_types": { - "name": "supported_types", - "kind": "attribute", - "path": "omniread.core.BaseParser.supported_types", - "signature": "", - "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." - }, - "content": { - "name": "content", - "kind": "attribute", - "path": "omniread.core.BaseParser.content", - "signature": "", - "docstring": null - }, - "parse": { - "name": "parse", - "kind": "function", - "path": "omniread.core.BaseParser.parse", - "signature": "", - "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." - }, - "supports": { - "name": "supports", - "kind": "function", - "path": "omniread.core.BaseParser.supports", - "signature": "", - "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." - } - } - }, - "BaseScraper": { - "name": "BaseScraper", - "kind": "class", - "path": "omniread.core.BaseScraper", - "signature": "", - "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", - "members": { - "fetch": { - "name": "fetch", - "kind": "function", - "path": "omniread.core.BaseScraper.fetch", - "signature": "", - "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." - } - } - }, - "content": { - "name": "content", - "kind": "module", - "path": "omniread.core.content", - "signature": null, - "docstring": "# Summary\n\nCanonical content models for OmniRead.\n\nThis module defines the **format-agnostic content representation** used across\nall parsers and scrapers in OmniRead.\n\nThe models defined here represent *what* was extracted, not *how* it was\nretrieved or parsed. Format-specific behavior and metadata must not alter\nthe semantic meaning of these models.", - "members": { - "Enum": { - "name": "Enum", - "kind": "alias", - "path": "omniread.core.content.Enum", - "signature": "", - "docstring": null - }, - "dataclass": { - "name": "dataclass", - "kind": "alias", - "path": "omniread.core.content.dataclass", - "signature": "", - "docstring": null - }, - "Any": { - "name": "Any", - "kind": "alias", - "path": "omniread.core.content.Any", - "signature": "", - "docstring": null - }, - "Mapping": { - "name": "Mapping", - "kind": "alias", - "path": "omniread.core.content.Mapping", - "signature": "", - "docstring": null - }, - "Optional": { - "name": "Optional", - "kind": "alias", - "path": "omniread.core.content.Optional", - "signature": "", - "docstring": null - }, - "ContentType": { - "name": "ContentType", - "kind": "class", - "path": "omniread.core.content.ContentType", - "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", - "members": { - "HTML": { - "name": "HTML", - "kind": "attribute", - "path": "omniread.core.content.ContentType.HTML", - "signature": null, - "docstring": "HTML document content." - }, - "PDF": { - "name": "PDF", - "kind": "attribute", - "path": "omniread.core.content.ContentType.PDF", - "signature": null, - "docstring": "PDF document content." - }, - "JSON": { - "name": "JSON", - "kind": "attribute", - "path": "omniread.core.content.ContentType.JSON", - "signature": null, - "docstring": "JSON document content." - }, - "XML": { - "name": "XML", - "kind": "attribute", - "path": "omniread.core.content.ContentType.XML", - "signature": null, - "docstring": "XML document content." - } - } - }, - "Content": { - "name": "Content", - "kind": "class", - "path": "omniread.core.content.Content", - "signature": "", - "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", - "members": { - "raw": { - "name": "raw", - "kind": "attribute", - "path": "omniread.core.content.Content.raw", - "signature": null, - "docstring": "Raw content bytes as retrieved from the source." - }, - "source": { - "name": "source", - "kind": "attribute", - "path": "omniread.core.content.Content.source", - "signature": null, - "docstring": "Identifier of the content origin (URL, file path, or logical name)." - }, - "content_type": { - "name": "content_type", - "kind": "attribute", - "path": "omniread.core.content.Content.content_type", - "signature": null, - "docstring": "Optional MIME type of the content, if known." - }, - "metadata": { - "name": "metadata", - "kind": "attribute", - "path": "omniread.core.content.Content.metadata", - "signature": null, - "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." - } - } - } - } - }, - "parser": { - "name": "parser", - "kind": "module", - "path": "omniread.core.parser", - "signature": null, - "docstring": "# Summary\n\nAbstract parsing contracts for OmniRead.\n\nThis module defines the **format-agnostic parser interface** used to transform\nraw content into structured, typed representations.\n\nParsers are responsible for:\n\n- Interpreting a single `Content` instance\n- Validating compatibility with the content type\n- Producing a structured output suitable for downstream consumers\n\nParsers are not responsible for:\n\n- Fetching or acquiring content\n- Performing retries or error recovery\n- Managing multiple content sources", - "members": { - "ABC": { - "name": "ABC", - "kind": "alias", - "path": "omniread.core.parser.ABC", - "signature": "", - "docstring": null - }, - "abstractmethod": { - "name": "abstractmethod", - "kind": "alias", - "path": "omniread.core.parser.abstractmethod", - "signature": "", - "docstring": null - }, - "Generic": { - "name": "Generic", - "kind": "alias", - "path": "omniread.core.parser.Generic", - "signature": "", - "docstring": null - }, - "TypeVar": { - "name": "TypeVar", - "kind": "alias", - "path": "omniread.core.parser.TypeVar", - "signature": "", - "docstring": null - }, - "Set": { - "name": "Set", - "kind": "alias", - "path": "omniread.core.parser.Set", - "signature": "", - "docstring": null - }, - "Content": { - "name": "Content", - "kind": "class", - "path": "omniread.core.parser.Content", - "signature": "", - "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", - "members": { - "raw": { - "name": "raw", - "kind": "attribute", - "path": "omniread.core.parser.Content.raw", - "signature": "", - "docstring": "Raw content bytes as retrieved from the source." - }, - "source": { - "name": "source", - "kind": "attribute", - "path": "omniread.core.parser.Content.source", - "signature": "", - "docstring": "Identifier of the content origin (URL, file path, or logical name)." - }, - "content_type": { - "name": "content_type", - "kind": "attribute", - "path": "omniread.core.parser.Content.content_type", - "signature": "", - "docstring": "Optional MIME type of the content, if known." - }, - "metadata": { - "name": "metadata", - "kind": "attribute", - "path": "omniread.core.parser.Content.metadata", - "signature": "", - "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." - } - } - }, - "ContentType": { - "name": "ContentType", - "kind": "class", - "path": "omniread.core.parser.ContentType", - "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", - "members": { - "HTML": { - "name": "HTML", - "kind": "attribute", - "path": "omniread.core.parser.ContentType.HTML", - "signature": "", - "docstring": "HTML document content." - }, - "PDF": { - "name": "PDF", - "kind": "attribute", - "path": "omniread.core.parser.ContentType.PDF", - "signature": "", - "docstring": "PDF document content." - }, - "JSON": { - "name": "JSON", - "kind": "attribute", - "path": "omniread.core.parser.ContentType.JSON", - "signature": "", - "docstring": "JSON document content." - }, - "XML": { - "name": "XML", - "kind": "attribute", - "path": "omniread.core.parser.ContentType.XML", - "signature": "", - "docstring": "XML document content." - } - } - }, - "T": { - "name": "T", - "kind": "attribute", - "path": "omniread.core.parser.T", - "signature": null, - "docstring": null - }, - "BaseParser": { - "name": "BaseParser", - "kind": "class", - "path": "omniread.core.parser.BaseParser", - "signature": "", - "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", - "members": { - "supported_types": { - "name": "supported_types", - "kind": "attribute", - "path": "omniread.core.parser.BaseParser.supported_types", - "signature": null, - "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." - }, - "content": { - "name": "content", - "kind": "attribute", - "path": "omniread.core.parser.BaseParser.content", - "signature": null, - "docstring": null - }, - "parse": { - "name": "parse", - "kind": "function", - "path": "omniread.core.parser.BaseParser.parse", - "signature": "", - "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." - }, - "supports": { - "name": "supports", - "kind": "function", - "path": "omniread.core.parser.BaseParser.supports", - "signature": "", - "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." - } - } - } - } - }, - "scraper": { - "name": "scraper", - "kind": "module", - "path": "omniread.core.scraper", - "signature": null, - "docstring": "# Summary\n\nAbstract scraping contracts for OmniRead.\n\nThis module defines the **format-agnostic scraper interface** responsible for\nacquiring raw content from external sources.\n\nScrapers are responsible for:\n\n- Locating and retrieving raw content bytes\n- Attaching minimal contextual metadata\n- Returning normalized `Content` objects\n\nScrapers are explicitly NOT responsible for:\n\n- Parsing or interpreting content\n- Inferring structure or semantics\n- Performing content-type specific processing\n\nAll interpretation must be delegated to parsers.", - "members": { - "ABC": { - "name": "ABC", - "kind": "alias", - "path": "omniread.core.scraper.ABC", - "signature": "", - "docstring": null - }, - "abstractmethod": { - "name": "abstractmethod", - "kind": "alias", - "path": "omniread.core.scraper.abstractmethod", - "signature": "", - "docstring": null - }, - "Any": { - "name": "Any", - "kind": "alias", - "path": "omniread.core.scraper.Any", - "signature": "", - "docstring": null - }, - "Mapping": { - "name": "Mapping", - "kind": "alias", - "path": "omniread.core.scraper.Mapping", - "signature": "", - "docstring": null - }, - "Optional": { - "name": "Optional", - "kind": "alias", - "path": "omniread.core.scraper.Optional", - "signature": "", - "docstring": null - }, - "Content": { - "name": "Content", - "kind": "class", - "path": "omniread.core.scraper.Content", - "signature": "", - "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", - "members": { - "raw": { - "name": "raw", - "kind": "attribute", - "path": "omniread.core.scraper.Content.raw", - "signature": "", - "docstring": "Raw content bytes as retrieved from the source." - }, - "source": { - "name": "source", - "kind": "attribute", - "path": "omniread.core.scraper.Content.source", - "signature": "", - "docstring": "Identifier of the content origin (URL, file path, or logical name)." - }, - "content_type": { - "name": "content_type", - "kind": "attribute", - "path": "omniread.core.scraper.Content.content_type", - "signature": "", - "docstring": "Optional MIME type of the content, if known." - }, - "metadata": { - "name": "metadata", - "kind": "attribute", - "path": "omniread.core.scraper.Content.metadata", - "signature": "", - "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." - } - } - }, - "BaseScraper": { - "name": "BaseScraper", - "kind": "class", - "path": "omniread.core.scraper.BaseScraper", - "signature": "", - "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", - "members": { - "fetch": { - "name": "fetch", - "kind": "function", - "path": "omniread.core.scraper.BaseScraper.fetch", - "signature": "", - "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." - } - } - } - } - } - } - }, - "html": { - "name": "html", - "kind": "module", - "path": "omniread.html", - "signature": null, - "docstring": "# Summary\n\nHTML format implementation for OmniRead.\n\nThis package provides **HTML-specific implementations** of the core OmniRead\ncontracts defined in `omniread.core`.\n\nIt includes:\n\n- HTML parsers that interpret HTML content.\n- HTML scrapers that retrieve HTML documents.\n\nKey characteristics:\n\n- Implements, but does not redefine, core contracts.\n- May contain HTML-specific behavior and edge-case handling.\n- Produces canonical content models defined in `omniread.core.content`.\n\nConsumers should depend on `omniread.core` interfaces wherever possible and\nuse this package only when HTML-specific behavior is required.\n\n---\n\n# Public API\n\n- `HTMLScraper`\n- `HTMLParser`\n\n---", - "members": { - "HTMLScraper": { - "name": "HTMLScraper", - "kind": "class", - "path": "omniread.html.HTMLScraper", - "signature": "", - "docstring": "Base HTML scraper using `httpx`.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns\n them as raw content wrapped in a `Content` object.\n - Fetches raw bytes and metadata only.\n - The scraper uses `httpx.Client` for HTTP requests, enforces an\n HTML content type, and preserves HTTP response metadata.\n\n **Constraints:**\n\n - The scraper does not: Parse HTML, perform retries or backoff,\n handle non-HTML responses.", - "members": { - "content_type": { - "name": "content_type", - "kind": "attribute", - "path": "omniread.html.HTMLScraper.content_type", - "signature": "", - "docstring": null - }, - "validate_content_type": { - "name": "validate_content_type", - "kind": "function", - "path": "omniread.html.HTMLScraper.validate_content_type", - "signature": "", - "docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content." - }, - "fetch": { - "name": "fetch", - "kind": "function", - "path": "omniread.html.HTMLScraper.fetch", - "signature": "", - "docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML." - } - } - }, - "HTMLParser": { - "name": "HTMLParser", - "kind": "class", - "path": "omniread.html.HTMLParser", - "signature": "", - "docstring": "Base HTML parser.\n\nNotes:\n **Responsibilities:**\n\n - This class extends the core `BaseParser` with HTML-specific behavior,\n including DOM parsing via BeautifulSoup and reusable extraction helpers.\n - Provides reusable helpers for HTML extraction. Concrete parsers must\n explicitly define the return type.\n\n **Guarantees:**\n\n - Accepts only HTML content.\n - Owns a parsed BeautifulSoup DOM tree.\n - Provides pure helper utilities for common HTML structures.\n\n **Constraints:**\n\n - Concrete subclasses must define the output type `T` and implement\n the `parse()` method.", - "members": { - "supported_types": { - "name": "supported_types", - "kind": "attribute", - "path": "omniread.html.HTMLParser.supported_types", - "signature": "", - "docstring": "Set of content types supported by this parser (HTML only)." - }, - "parse": { - "name": "parse", - "kind": "function", - "path": "omniread.html.HTMLParser.parse", - "signature": "", - "docstring": "Fully parse the HTML content into structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the HTML DOM and return a\n deterministic, structured output." - }, - "parse_div": { - "name": "parse_div", - "kind": "function", - "path": "omniread.html.HTMLParser.parse_div", - "signature": "", - "docstring": "Extract normalized text from a `
` element.\n\nArgs:\n div (Tag):\n BeautifulSoup tag representing a `
`.\n\nReturns:\n list[list[str]]:\n A list of rows, where each row is a list of cell text values." - }, - "parse_meta": { - "name": "parse_meta", - "kind": "function", - "path": "omniread.html.HTMLParser.parse_meta", - "signature": "", - "docstring": "Extract high-level metadata from the HTML document.\n\nReturns:\n dict[str, Any]:\n Dictionary containing extracted metadata.\n\nNotes:\n **Responsibilities:**\n\n - Extract high-level metadata from the HTML document.\n - This includes: Document title, `` tag name/property to\n content mappings." - } - } - }, - "parser": { - "name": "parser", - "kind": "module", - "path": "omniread.html.parser", - "signature": null, - "docstring": "# Summary\n\nHTML parser base implementations for OmniRead.\n\nThis module provides reusable HTML parsing utilities built on top of\nthe abstract parser contracts defined in `omniread.core.parser`.\n\nIt supplies:\n\n- Content-type enforcement for HTML inputs\n- BeautifulSoup initialization and lifecycle management\n- Common helper methods for extracting structured data from HTML elements\n\nConcrete parsers must subclass `HTMLParser` and implement the `parse()` method\nto return a structured representation appropriate for their use case.", - "members": { - "Any": { - "name": "Any", - "kind": "alias", - "path": "omniread.html.parser.Any", - "signature": "", - "docstring": null - }, - "Generic": { - "name": "Generic", - "kind": "alias", - "path": "omniread.html.parser.Generic", - "signature": "", - "docstring": null - }, - "TypeVar": { - "name": "TypeVar", - "kind": "alias", - "path": "omniread.html.parser.TypeVar", - "signature": "", - "docstring": null - }, - "Optional": { - "name": "Optional", - "kind": "alias", - "path": "omniread.html.parser.Optional", - "signature": "", - "docstring": null - }, - "abstractmethod": { - "name": "abstractmethod", - "kind": "alias", - "path": "omniread.html.parser.abstractmethod", - "signature": "", - "docstring": null - }, - "BeautifulSoup": { - "name": "BeautifulSoup", - "kind": "alias", - "path": "omniread.html.parser.BeautifulSoup", - "signature": "", - "docstring": null - }, - "Tag": { - "name": "Tag", - "kind": "alias", - "path": "omniread.html.parser.Tag", - "signature": "", - "docstring": null - }, - "ContentType": { - "name": "ContentType", - "kind": "class", - "path": "omniread.html.parser.ContentType", - "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", - "members": { - "HTML": { - "name": "HTML", - "kind": "attribute", - "path": "omniread.html.parser.ContentType.HTML", - "signature": "", - "docstring": "HTML document content." - }, - "PDF": { - "name": "PDF", - "kind": "attribute", - "path": "omniread.html.parser.ContentType.PDF", - "signature": "", - "docstring": "PDF document content." - }, - "JSON": { - "name": "JSON", - "kind": "attribute", - "path": "omniread.html.parser.ContentType.JSON", - "signature": "", - "docstring": "JSON document content." - }, - "XML": { - "name": "XML", - "kind": "attribute", - "path": "omniread.html.parser.ContentType.XML", - "signature": "", - "docstring": "XML document content." - } - } - }, - "Content": { - "name": "Content", - "kind": "class", - "path": "omniread.html.parser.Content", - "signature": "", - "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", - "members": { - "raw": { - "name": "raw", - "kind": "attribute", - "path": "omniread.html.parser.Content.raw", - "signature": "", - "docstring": "Raw content bytes as retrieved from the source." - }, - "source": { - "name": "source", - "kind": "attribute", - "path": "omniread.html.parser.Content.source", - "signature": "", - "docstring": "Identifier of the content origin (URL, file path, or logical name)." - }, - "content_type": { - "name": "content_type", - "kind": "attribute", - "path": "omniread.html.parser.Content.content_type", - "signature": "", - "docstring": "Optional MIME type of the content, if known." - }, - "metadata": { - "name": "metadata", - "kind": "attribute", - "path": "omniread.html.parser.Content.metadata", - "signature": "", - "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." - } - } - }, - "BaseParser": { - "name": "BaseParser", - "kind": "class", - "path": "omniread.html.parser.BaseParser", - "signature": "", - "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", - "members": { - "supported_types": { - "name": "supported_types", - "kind": "attribute", - "path": "omniread.html.parser.BaseParser.supported_types", - "signature": "", - "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." - }, - "content": { - "name": "content", - "kind": "attribute", - "path": "omniread.html.parser.BaseParser.content", - "signature": "", - "docstring": null - }, - "parse": { - "name": "parse", - "kind": "function", - "path": "omniread.html.parser.BaseParser.parse", - "signature": "", - "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." - }, - "supports": { - "name": "supports", - "kind": "function", - "path": "omniread.html.parser.BaseParser.supports", - "signature": "", - "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." - } - } - }, - "T": { - "name": "T", - "kind": "attribute", - "path": "omniread.html.parser.T", - "signature": null, - "docstring": null - }, - "HTMLParser": { - "name": "HTMLParser", - "kind": "class", - "path": "omniread.html.parser.HTMLParser", - "signature": "", - "docstring": "Base HTML parser.\n\nNotes:\n **Responsibilities:**\n\n - This class extends the core `BaseParser` with HTML-specific behavior,\n including DOM parsing via BeautifulSoup and reusable extraction helpers.\n - Provides reusable helpers for HTML extraction. Concrete parsers must\n explicitly define the return type.\n\n **Guarantees:**\n\n - Accepts only HTML content.\n - Owns a parsed BeautifulSoup DOM tree.\n - Provides pure helper utilities for common HTML structures.\n\n **Constraints:**\n\n - Concrete subclasses must define the output type `T` and implement\n the `parse()` method.", - "members": { - "supported_types": { - "name": "supported_types", - "kind": "attribute", - "path": "omniread.html.parser.HTMLParser.supported_types", - "signature": null, - "docstring": "Set of content types supported by this parser (HTML only)." - }, - "parse": { - "name": "parse", - "kind": "function", - "path": "omniread.html.parser.HTMLParser.parse", - "signature": "", - "docstring": "Fully parse the HTML content into structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the HTML DOM and return a\n deterministic, structured output." - }, - "parse_div": { - "name": "parse_div", - "kind": "function", - "path": "omniread.html.parser.HTMLParser.parse_div", - "signature": "", - "docstring": "Extract normalized text from a `
` element.\n\nArgs:\n div (Tag):\n BeautifulSoup tag representing a `
`.\n\nReturns:\n list[list[str]]:\n A list of rows, where each row is a list of cell text values." - }, - "parse_meta": { - "name": "parse_meta", - "kind": "function", - "path": "omniread.html.parser.HTMLParser.parse_meta", - "signature": "", - "docstring": "Extract high-level metadata from the HTML document.\n\nReturns:\n dict[str, Any]:\n Dictionary containing extracted metadata.\n\nNotes:\n **Responsibilities:**\n\n - Extract high-level metadata from the HTML document.\n - This includes: Document title, `` tag name/property to\n content mappings." - } - } - }, - "list": { - "name": "list", - "kind": "alias", - "path": "omniread.html.parser.list", - "signature": "", - "docstring": null - }, - "dict": { - "name": "dict", - "kind": "alias", - "path": "omniread.html.parser.dict", - "signature": "", - "docstring": null - } - } - }, - "scraper": { - "name": "scraper", - "kind": "module", - "path": "omniread.html.scraper", - "signature": null, - "docstring": "# Summary\n\nHTML scraping implementation for OmniRead.\n\nThis module provides an HTTP-based scraper for retrieving HTML documents.\nIt implements the core `BaseScraper` contract using `httpx` as the transport\nlayer.\n\nThis scraper is responsible for:\n\n- Fetching raw HTML bytes over HTTP(S)\n- Validating response content type\n- Attaching HTTP metadata to the returned content\n\nThis scraper is not responsible for:\n\n- Parsing or interpreting HTML\n- Retrying failed requests\n- Managing crawl policies or rate limiting", - "members": { - "httpx": { - "name": "httpx", - "kind": "alias", - "path": "omniread.html.scraper.httpx", - "signature": "", - "docstring": null - }, - "Any": { - "name": "Any", - "kind": "alias", - "path": "omniread.html.scraper.Any", - "signature": "", - "docstring": null - }, - "Mapping": { - "name": "Mapping", - "kind": "alias", - "path": "omniread.html.scraper.Mapping", - "signature": "", - "docstring": null - }, - "Optional": { - "name": "Optional", - "kind": "alias", - "path": "omniread.html.scraper.Optional", - "signature": "", - "docstring": null - }, - "Content": { - "name": "Content", - "kind": "class", - "path": "omniread.html.scraper.Content", - "signature": "", - "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", - "members": { - "raw": { - "name": "raw", - "kind": "attribute", - "path": "omniread.html.scraper.Content.raw", - "signature": "", - "docstring": "Raw content bytes as retrieved from the source." - }, - "source": { - "name": "source", - "kind": "attribute", - "path": "omniread.html.scraper.Content.source", - "signature": "", - "docstring": "Identifier of the content origin (URL, file path, or logical name)." - }, - "content_type": { - "name": "content_type", - "kind": "attribute", - "path": "omniread.html.scraper.Content.content_type", - "signature": "", - "docstring": "Optional MIME type of the content, if known." - }, - "metadata": { - "name": "metadata", - "kind": "attribute", - "path": "omniread.html.scraper.Content.metadata", - "signature": "", - "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." - } - } - }, - "ContentType": { - "name": "ContentType", - "kind": "class", - "path": "omniread.html.scraper.ContentType", - "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", - "members": { - "HTML": { - "name": "HTML", - "kind": "attribute", - "path": "omniread.html.scraper.ContentType.HTML", - "signature": "", - "docstring": "HTML document content." - }, - "PDF": { - "name": "PDF", - "kind": "attribute", - "path": "omniread.html.scraper.ContentType.PDF", - "signature": "", - "docstring": "PDF document content." - }, - "JSON": { - "name": "JSON", - "kind": "attribute", - "path": "omniread.html.scraper.ContentType.JSON", - "signature": "", - "docstring": "JSON document content." - }, - "XML": { - "name": "XML", - "kind": "attribute", - "path": "omniread.html.scraper.ContentType.XML", - "signature": "", - "docstring": "XML document content." - } - } - }, - "BaseScraper": { - "name": "BaseScraper", - "kind": "class", - "path": "omniread.html.scraper.BaseScraper", - "signature": "", - "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", - "members": { - "fetch": { - "name": "fetch", - "kind": "function", - "path": "omniread.html.scraper.BaseScraper.fetch", - "signature": "", - "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." - } - } - }, - "HTMLScraper": { - "name": "HTMLScraper", - "kind": "class", - "path": "omniread.html.scraper.HTMLScraper", - "signature": "", - "docstring": "Base HTML scraper using `httpx`.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns\n them as raw content wrapped in a `Content` object.\n - Fetches raw bytes and metadata only.\n - The scraper uses `httpx.Client` for HTTP requests, enforces an\n HTML content type, and preserves HTTP response metadata.\n\n **Constraints:**\n\n - The scraper does not: Parse HTML, perform retries or backoff,\n handle non-HTML responses.", - "members": { - "content_type": { - "name": "content_type", - "kind": "attribute", - "path": "omniread.html.scraper.HTMLScraper.content_type", - "signature": null, - "docstring": null - }, - "validate_content_type": { - "name": "validate_content_type", - "kind": "function", - "path": "omniread.html.scraper.HTMLScraper.validate_content_type", - "signature": "", - "docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content." - }, - "fetch": { - "name": "fetch", - "kind": "function", - "path": "omniread.html.scraper.HTMLScraper.fetch", - "signature": "", - "docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML." - } - } - } - } - } - } - }, - "pdf": { - "name": "pdf", - "kind": "module", - "path": "omniread.pdf", - "signature": null, - "docstring": "# Summary\n\nPDF format implementation for OmniRead.\n\nThis package provides **PDF-specific implementations** of the core OmniRead\ncontracts defined in `omniread.core`.\n\nUnlike HTML, PDF handling requires an explicit client layer for document\naccess. This package therefore includes:\n\n- PDF clients for acquiring raw PDF data.\n- PDF scrapers that coordinate client access.\n- PDF parsers that extract structured content from PDF binaries.\n\nPublic exports from this package represent the supported PDF pipeline\nand are safe for consumers to import directly when working with PDFs.\n\n---\n\n# Public API\n\n- `FileSystemPDFClient`\n- `PDFScraper`\n- `PDFParser`\n\n---", - "members": { - "FileSystemPDFClient": { - "name": "FileSystemPDFClient", - "kind": "class", - "path": "omniread.pdf.FileSystemPDFClient", - "signature": "", - "docstring": "PDF client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads PDF files directly from the disk and returns\n their raw binary contents.", - "members": { - "fetch": { - "name": "fetch", - "kind": "function", - "path": "omniread.pdf.FileSystemPDFClient.fetch", - "signature": "", - "docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the PDF file.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." - } - } - }, - "PDFScraper": { - "name": "PDFScraper", - "kind": "class", - "path": "omniread.pdf.PDFScraper", - "signature": "", - "docstring": "Scraper for PDF sources.\n\nNotes:\n **Responsibilities:**\n\n - Delegates byte retrieval to a PDF client and normalizes output\n into `Content`.\n - Preserves caller-provided metadata.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", - "members": { - "fetch": { - "name": "fetch", - "kind": "function", - "path": "omniread.pdf.PDFScraper.fetch", - "signature": "", - "docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client." - } - } - }, - "PDFParser": { - "name": "PDFParser", - "kind": "class", - "path": "omniread.pdf.PDFParser", - "signature": "", - "docstring": "Base PDF parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces PDF content-type compatibility and provides\n the extension point for implementing concrete PDF parsing strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", - "members": { - "supported_types": { - "name": "supported_types", - "kind": "attribute", - "path": "omniread.pdf.PDFParser.supported_types", - "signature": "", - "docstring": "Set of content types supported by this parser (PDF only)." - }, - "parse": { - "name": "parse", - "kind": "function", - "path": "omniread.pdf.PDFParser.parse", - "signature": "", - "docstring": "Parse PDF content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the PDF binary payload and\n return a deterministic, structured output." - } - } - }, - "client": { - "name": "client", - "kind": "module", - "path": "omniread.pdf.client", - "signature": null, - "docstring": "# Summary\n\nPDF client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw PDF\nbytes from a concrete backing store.\n\nClients provide low-level access to PDF binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems", - "members": { - "Any": { - "name": "Any", - "kind": "alias", - "path": "omniread.pdf.client.Any", - "signature": "", - "docstring": null - }, - "ABC": { - "name": "ABC", - "kind": "alias", - "path": "omniread.pdf.client.ABC", - "signature": "", - "docstring": null - }, - "abstractmethod": { - "name": "abstractmethod", - "kind": "alias", - "path": "omniread.pdf.client.abstractmethod", - "signature": "", - "docstring": null - }, - "Path": { - "name": "Path", - "kind": "alias", - "path": "omniread.pdf.client.Path", - "signature": "", - "docstring": null - }, - "BasePDFClient": { - "name": "BasePDFClient", - "kind": "class", - "path": "omniread.pdf.client.BasePDFClient", - "signature": "", - "docstring": "Abstract client responsible for retrieving PDF bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full PDF binary payload.\n - Raise retrieval-specific errors on failure.", - "members": { - "fetch": { - "name": "fetch", - "kind": "function", - "path": "omniread.pdf.client.BasePDFClient.fetch", - "signature": "", - "docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF location, such as a file path, object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." - } - } - }, - "FileSystemPDFClient": { - "name": "FileSystemPDFClient", - "kind": "class", - "path": "omniread.pdf.client.FileSystemPDFClient", - "signature": "", - "docstring": "PDF client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads PDF files directly from the disk and returns\n their raw binary contents.", - "members": { - "fetch": { - "name": "fetch", - "kind": "function", - "path": "omniread.pdf.client.FileSystemPDFClient.fetch", - "signature": "", - "docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the PDF file.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." - } - } - } - } - }, - "parser": { - "name": "parser", - "kind": "module", - "path": "omniread.pdf.parser", - "signature": null, - "docstring": "# Summary\n\nPDF parser base implementations for OmniRead.\n\nThis module defines the **PDF-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for PDF content.\n\nPDF parsers are responsible for interpreting binary PDF data and producing\nstructured representations suitable for downstream consumption.", - "members": { - "Generic": { - "name": "Generic", - "kind": "alias", - "path": "omniread.pdf.parser.Generic", - "signature": "", - "docstring": null - }, - "TypeVar": { - "name": "TypeVar", - "kind": "alias", - "path": "omniread.pdf.parser.TypeVar", - "signature": "", - "docstring": null - }, - "abstractmethod": { - "name": "abstractmethod", - "kind": "alias", - "path": "omniread.pdf.parser.abstractmethod", - "signature": "", - "docstring": null - }, - "ContentType": { - "name": "ContentType", - "kind": "class", - "path": "omniread.pdf.parser.ContentType", - "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", - "members": { - "HTML": { - "name": "HTML", - "kind": "attribute", - "path": "omniread.pdf.parser.ContentType.HTML", - "signature": "", - "docstring": "HTML document content." - }, - "PDF": { - "name": "PDF", - "kind": "attribute", - "path": "omniread.pdf.parser.ContentType.PDF", - "signature": "", - "docstring": "PDF document content." - }, - "JSON": { - "name": "JSON", - "kind": "attribute", - "path": "omniread.pdf.parser.ContentType.JSON", - "signature": "", - "docstring": "JSON document content." - }, - "XML": { - "name": "XML", - "kind": "attribute", - "path": "omniread.pdf.parser.ContentType.XML", - "signature": "", - "docstring": "XML document content." - } - } - }, - "BaseParser": { - "name": "BaseParser", - "kind": "class", - "path": "omniread.pdf.parser.BaseParser", - "signature": "", - "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", - "members": { - "supported_types": { - "name": "supported_types", - "kind": "attribute", - "path": "omniread.pdf.parser.BaseParser.supported_types", - "signature": "", - "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." - }, - "content": { - "name": "content", - "kind": "attribute", - "path": "omniread.pdf.parser.BaseParser.content", - "signature": "", - "docstring": null - }, - "parse": { - "name": "parse", - "kind": "function", - "path": "omniread.pdf.parser.BaseParser.parse", - "signature": "", - "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." - }, - "supports": { - "name": "supports", - "kind": "function", - "path": "omniread.pdf.parser.BaseParser.supports", - "signature": "", - "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." - } - } - }, - "T": { - "name": "T", - "kind": "attribute", - "path": "omniread.pdf.parser.T", - "signature": null, - "docstring": null - }, - "PDFParser": { - "name": "PDFParser", - "kind": "class", - "path": "omniread.pdf.parser.PDFParser", - "signature": "", - "docstring": "Base PDF parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces PDF content-type compatibility and provides\n the extension point for implementing concrete PDF parsing strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", - "members": { - "supported_types": { - "name": "supported_types", - "kind": "attribute", - "path": "omniread.pdf.parser.PDFParser.supported_types", - "signature": null, - "docstring": "Set of content types supported by this parser (PDF only)." - }, - "parse": { - "name": "parse", - "kind": "function", - "path": "omniread.pdf.parser.PDFParser.parse", - "signature": "", - "docstring": "Parse PDF content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the PDF binary payload and\n return a deterministic, structured output." - } - } - } - } - }, - "scraper": { - "name": "scraper", - "kind": "module", - "path": "omniread.pdf.scraper", - "signature": null, - "docstring": "# Summary\n\nPDF scraping implementation for OmniRead.\n\nThis module provides a PDF-specific scraper that coordinates PDF byte\nretrieval via a client and normalizes the result into a `Content` object.\n\nThe scraper implements the core `BaseScraper` contract while delegating\nall storage and access concerns to a `BasePDFClient` implementation.", - "members": { - "Any": { - "name": "Any", - "kind": "alias", - "path": "omniread.pdf.scraper.Any", - "signature": "", - "docstring": null - }, - "Mapping": { - "name": "Mapping", - "kind": "alias", - "path": "omniread.pdf.scraper.Mapping", - "signature": "", - "docstring": null - }, - "Optional": { - "name": "Optional", - "kind": "alias", - "path": "omniread.pdf.scraper.Optional", - "signature": "", - "docstring": null - }, - "Content": { - "name": "Content", - "kind": "class", - "path": "omniread.pdf.scraper.Content", - "signature": "", - "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", - "members": { - "raw": { - "name": "raw", - "kind": "attribute", - "path": "omniread.pdf.scraper.Content.raw", - "signature": "", - "docstring": "Raw content bytes as retrieved from the source." - }, - "source": { - "name": "source", - "kind": "attribute", - "path": "omniread.pdf.scraper.Content.source", - "signature": "", - "docstring": "Identifier of the content origin (URL, file path, or logical name)." - }, - "content_type": { - "name": "content_type", - "kind": "attribute", - "path": "omniread.pdf.scraper.Content.content_type", - "signature": "", - "docstring": "Optional MIME type of the content, if known." - }, - "metadata": { - "name": "metadata", - "kind": "attribute", - "path": "omniread.pdf.scraper.Content.metadata", - "signature": "", - "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." - } - } - }, - "ContentType": { - "name": "ContentType", - "kind": "class", - "path": "omniread.pdf.scraper.ContentType", - "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", - "members": { - "HTML": { - "name": "HTML", - "kind": "attribute", - "path": "omniread.pdf.scraper.ContentType.HTML", - "signature": "", - "docstring": "HTML document content." - }, - "PDF": { - "name": "PDF", - "kind": "attribute", - "path": "omniread.pdf.scraper.ContentType.PDF", - "signature": "", - "docstring": "PDF document content." - }, - "JSON": { - "name": "JSON", - "kind": "attribute", - "path": "omniread.pdf.scraper.ContentType.JSON", - "signature": "", - "docstring": "JSON document content." - }, - "XML": { - "name": "XML", - "kind": "attribute", - "path": "omniread.pdf.scraper.ContentType.XML", - "signature": "", - "docstring": "XML document content." - } - } - }, - "BaseScraper": { - "name": "BaseScraper", - "kind": "class", - "path": "omniread.pdf.scraper.BaseScraper", - "signature": "", - "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", - "members": { - "fetch": { - "name": "fetch", - "kind": "function", - "path": "omniread.pdf.scraper.BaseScraper.fetch", - "signature": "", - "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." - } - } - }, - "BasePDFClient": { - "name": "BasePDFClient", - "kind": "class", - "path": "omniread.pdf.scraper.BasePDFClient", - "signature": "", - "docstring": "Abstract client responsible for retrieving PDF bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full PDF binary payload.\n - Raise retrieval-specific errors on failure.", - "members": { - "fetch": { - "name": "fetch", - "kind": "function", - "path": "omniread.pdf.scraper.BasePDFClient.fetch", - "signature": "", - "docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF location, such as a file path, object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." - } - } - }, - "PDFScraper": { - "name": "PDFScraper", - "kind": "class", - "path": "omniread.pdf.scraper.PDFScraper", - "signature": "", - "docstring": "Scraper for PDF sources.\n\nNotes:\n **Responsibilities:**\n\n - Delegates byte retrieval to a PDF client and normalizes output\n into `Content`.\n - Preserves caller-provided metadata.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", - "members": { - "fetch": { - "name": "fetch", - "kind": "function", - "path": "omniread.pdf.scraper.PDFScraper.fetch", - "signature": "", - "docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client." - } - } - } - } - } - } - } - } - } -} \ No newline at end of file diff --git a/mcp_docs/nav.json b/mcp_docs/nav.json deleted file mode 100644 index 61b75cc..0000000 --- a/mcp_docs/nav.json +++ /dev/null @@ -1,50 +0,0 @@ -[ - { - "module": "omniread", - "resource": "doc://modules/omniread" - }, - { - "module": "omniread.core", - "resource": "doc://modules/omniread.core" - }, - { - "module": "omniread.core.content", - "resource": "doc://modules/omniread.core.content" - }, - { - "module": "omniread.core.parser", - "resource": "doc://modules/omniread.core.parser" - }, - { - "module": "omniread.core.scraper", - "resource": "doc://modules/omniread.core.scraper" - }, - { - "module": "omniread.html", - "resource": "doc://modules/omniread.html" - }, - { - "module": "omniread.html.parser", - "resource": "doc://modules/omniread.html.parser" - }, - { - "module": "omniread.html.scraper", - "resource": "doc://modules/omniread.html.scraper" - }, - { - "module": "omniread.pdf", - "resource": "doc://modules/omniread.pdf" - }, - { - "module": "omniread.pdf.client", - "resource": "doc://modules/omniread.pdf.client" - }, - { - "module": "omniread.pdf.parser", - "resource": "doc://modules/omniread.pdf.parser" - }, - { - "module": "omniread.pdf.scraper", - "resource": "doc://modules/omniread.pdf.scraper" - } -] \ No newline at end of file diff --git a/omniread/__init__.py b/omniread/__init__.py index 46f53da..0875478 100644 --- a/omniread/__init__.py +++ b/omniread/__init__.py @@ -132,23 +132,19 @@ __all__ = [ # core "Content", "ContentType", - # html "HTMLScraper", "HTMLParser", - # pdf "FileSystemPDFClient", "PDFScraper", "PDFParser", - # csv "BaseCsvClient", "FileSystemCsvClient", "CsvScraper", "CsvParser", "CsvParserBase", - # xlsx "BaseXlsxClient", "FileSystemXlsxClient", diff --git a/omniread/__init__.pyi b/omniread/__init__.pyi index 0980e93..3699810 100644 --- a/omniread/__init__.pyi +++ b/omniread/__init__.pyi @@ -1,6 +1,6 @@ from .core import Content, ContentType -from .html import HTMLScraper, HTMLParser -from .pdf import FileSystemPDFClient, PDFScraper, PDFParser +from .html import HTMLParser, HTMLScraper +from .pdf import FileSystemPDFClient, PDFParser, PDFScraper __all__ = [ "Content", diff --git a/omniread/core/content.py b/omniread/core/content.py index 775a843..21074b5 100644 --- a/omniread/core/content.py +++ b/omniread/core/content.py @@ -11,9 +11,10 @@ retrieved or parsed. Format-specific behavior and metadata must not alter the semantic meaning of these models. """ -from enum import Enum +from collections.abc import Mapping from dataclasses import dataclass -from typing import Any, Mapping, Optional +from enum import Enum +from typing import Any class ContentType(str, Enum): @@ -72,12 +73,12 @@ class Content: Identifier of the content origin (URL, file path, or logical name). """ - content_type: Optional[ContentType] = None + content_type: ContentType | None = None """ Optional MIME type of the content, if known. """ - metadata: Optional[Mapping[str, Any]] = None + metadata: Mapping[str, Any] | None = None """ Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes). """ diff --git a/omniread/core/content.pyi b/omniread/core/content.pyi index 5606462..25f6081 100644 --- a/omniread/core/content.pyi +++ b/omniread/core/content.pyi @@ -1,15 +1,24 @@ +from collections.abc import Mapping from enum import Enum -from typing import Any, Mapping, Optional +from typing import Any class ContentType(str, Enum): HTML = "text/html" PDF = "application/pdf" + XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + CSV = "text/csv" JSON = "application/json" XML = "application/xml" class Content: raw: bytes source: str - content_type: Optional[ContentType] - metadata: Optional[Mapping[str, Any]] - def __init__(self, raw: bytes, source: str, content_type: Optional[ContentType] = ..., metadata: Optional[Mapping[str, Any]] = ...) -> None: ... + content_type: ContentType | None + metadata: Mapping[str, Any] | None + def __init__( + self, + raw: bytes, + source: str, + content_type: ContentType | None = ..., + metadata: Mapping[str, Any] | None = ..., + ) -> None: ... diff --git a/omniread/core/parser.py b/omniread/core/parser.py index 7c32993..32baeb2 100644 --- a/omniread/core/parser.py +++ b/omniread/core/parser.py @@ -20,7 +20,7 @@ Parsers are not responsible for: """ from abc import ABC, abstractmethod -from typing import Generic, TypeVar, Set +from typing import Generic, TypeVar from .content import Content, ContentType @@ -46,7 +46,7 @@ class BaseParser(ABC, Generic[T]): - Implementations must remain deterministic for a given input. """ - supported_types: Set[ContentType] = set() + supported_types: set[ContentType] = set() """ Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic. """ diff --git a/omniread/core/parser.pyi b/omniread/core/parser.pyi index c76e60f..a731065 100644 --- a/omniread/core/parser.pyi +++ b/omniread/core/parser.pyi @@ -1,11 +1,12 @@ from abc import ABC, abstractmethod -from typing import Generic, TypeVar, Set +from typing import Generic, TypeVar + from .content import Content, ContentType T = TypeVar("T") class BaseParser(ABC, Generic[T]): - supported_types: Set[ContentType] + supported_types: set[ContentType] content: Content def __init__(self, content: Content) -> None: ... @abstractmethod diff --git a/omniread/core/scraper.py b/omniread/core/scraper.py index fd4bd81..289d767 100644 --- a/omniread/core/scraper.py +++ b/omniread/core/scraper.py @@ -22,7 +22,8 @@ All interpretation must be delegated to parsers. """ from abc import ABC, abstractmethod -from typing import Any, Mapping, Optional +from collections.abc import Mapping +from typing import Any from .content import Content @@ -53,7 +54,7 @@ class BaseScraper(ABC): self, source: str, *, - metadata: Optional[Mapping[str, Any]] = None, + metadata: Mapping[str, Any] | None = None, ) -> Content: """ Fetch raw content from the given source. @@ -62,7 +63,7 @@ class BaseScraper(ABC): source (str): Location identifier (URL, file path, S3 URI, etc.). - metadata (Optional[Mapping[str, Any]], optional): + metadata (Mapping[str, Any] | None, optional): Optional hints for the scraper (headers, auth, etc.). Returns: diff --git a/omniread/core/scraper.pyi b/omniread/core/scraper.pyi index cf0ee5f..18d4405 100644 --- a/omniread/core/scraper.pyi +++ b/omniread/core/scraper.pyi @@ -1,7 +1,11 @@ from abc import ABC, abstractmethod -from typing import Any, Mapping, Optional +from collections.abc import Mapping +from typing import Any + from .content import Content class BaseScraper(ABC): @abstractmethod - def fetch(self, source: str, *, metadata: Optional[Mapping[str, Any]] = ...) -> Content: ... + def fetch( + self, source: str, *, metadata: Mapping[str, Any] | None = ... + ) -> Content: ... diff --git a/omniread/csv/__init__.pyi b/omniread/csv/__init__.pyi new file mode 100644 index 0000000..500c89e --- /dev/null +++ b/omniread/csv/__init__.pyi @@ -0,0 +1,12 @@ +from .client import BaseCsvClient, FileSystemCsvClient +from .parser import CsvParser +from .parser_base import CsvParserBase +from .scraper import CsvScraper + +__all__ = [ + "BaseCsvClient", + "FileSystemCsvClient", + "CsvScraper", + "CsvParser", + "CsvParserBase", +] diff --git a/omniread/csv/client.py b/omniread/csv/client.py index 22ad75f..e373b81 100644 --- a/omniread/csv/client.py +++ b/omniread/csv/client.py @@ -17,9 +17,9 @@ Typical backing stores include: - Network file systems """ -from typing import Any from abc import ABC, abstractmethod from pathlib import Path +from typing import Any class BaseCsvClient(ABC): diff --git a/omniread/csv/client.pyi b/omniread/csv/client.pyi new file mode 100644 index 0000000..0df163e --- /dev/null +++ b/omniread/csv/client.pyi @@ -0,0 +1,10 @@ +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Any + +class BaseCsvClient(ABC): + @abstractmethod + def fetch(self, source: Any) -> bytes: ... + +class FileSystemCsvClient(BaseCsvClient): + def fetch(self, path: Path) -> bytes: ... diff --git a/omniread/csv/parser.py b/omniread/csv/parser.py index 358e664..36857f7 100644 --- a/omniread/csv/parser.py +++ b/omniread/csv/parser.py @@ -13,15 +13,15 @@ detection or column interpretation beyond basic cell normalization and delimiter detection. """ -from io import StringIO from csv import Sniffer, reader -from typing import List +from io import StringIO from omniread.core.content import Content + from .parser_base import CsvParserBase -class CsvParser(CsvParserBase): +class CsvParser(CsvParserBase[list[list[str]]]): """ Generic csv parser producing string rows from the document. @@ -54,17 +54,17 @@ class CsvParser(CsvParserBase): """ super().__init__(content) - def parse(self) -> List[List[str]]: + def parse(self) -> list[list[str]]: """ Parse the document into normalized string rows. Returns: - List[List[str]]: + list[list[str]]: Rows of the document. """ return self.rows() - def rows(self, *, skip_empty: bool = True) -> List[List[str]]: + def rows(self, *, skip_empty: bool = True) -> list[list[str]]: """ Extract normalized string rows from the document. @@ -74,7 +74,7 @@ class CsvParser(CsvParserBase): omitted. Returns: - List[List[str]]: + list[list[str]]: Normalized rows; trailing blank cells are trimmed per row. """ text = self._decode(self.content.raw) @@ -84,7 +84,7 @@ class CsvParser(CsvParserBase): delimiter = dialect.delimiter except Exception: delimiter = "," - out: List[List[str]] = [] + out: list[list[str]] = [] for row in reader(StringIO(text), delimiter=delimiter): cells = [c.strip() for c in row] while cells and not cells[-1]: diff --git a/omniread/csv/parser.pyi b/omniread/csv/parser.pyi new file mode 100644 index 0000000..7d75b7b --- /dev/null +++ b/omniread/csv/parser.pyi @@ -0,0 +1,8 @@ +from omniread.core.content import Content + +from .parser_base import CsvParserBase + +class CsvParser(CsvParserBase[list[list[str]]]): + def __init__(self, content: Content) -> None: ... + def parse(self) -> list[list[str]]: ... + def rows(self, *, skip_empty: bool = True) -> list[list[str]]: ... diff --git a/omniread/csv/parser_base.py b/omniread/csv/parser_base.py index 5daf885..cfa9ec9 100644 --- a/omniread/csv/parser_base.py +++ b/omniread/csv/parser_base.py @@ -8,8 +8,8 @@ format-agnostic `BaseParser` with constraints appropriate for comma-separated-value documents. """ -from typing import Generic, TypeVar from abc import abstractmethod +from typing import Generic, TypeVar from omniread.core.content import ContentType from omniread.core.parser import BaseParser diff --git a/omniread/csv/parser_base.pyi b/omniread/csv/parser_base.pyi new file mode 100644 index 0000000..fdbcaa5 --- /dev/null +++ b/omniread/csv/parser_base.pyi @@ -0,0 +1,13 @@ +from abc import abstractmethod +from typing import Generic, TypeVar + +from omniread.core.content import ContentType +from omniread.core.parser import BaseParser + +T = TypeVar("T") + +class CsvParserBase(BaseParser[T], Generic[T]): + supported_types: set[ContentType] + + @abstractmethod + def parse(self) -> T: ... diff --git a/omniread/csv/scraper.py b/omniread/csv/scraper.py index 5785dfc..3b0671b 100644 --- a/omniread/csv/scraper.py +++ b/omniread/csv/scraper.py @@ -11,9 +11,11 @@ The scraper does not interpret or parse the acquired bytes; it wraps them in the canonical `Content` model. """ -from typing import Any, Mapping, Optional +from collections.abc import Mapping +from typing import Any from omniread.core.content import Content, ContentType + from .client import BaseCsvClient @@ -48,7 +50,7 @@ class CsvScraper: self, source: Any, *, - metadata: Optional[Mapping[str, Any]] = None, + metadata: Mapping[str, Any] | None = None, ) -> Content: """ Fetch a csv document from the given source. @@ -57,7 +59,7 @@ class CsvScraper: source (Any): Identifier of the csv source as understood by the configured client. - metadata (Optional[Mapping[str, Any]], optional): + metadata (Mapping[str, Any] | None, optional): Optional metadata to attach to the returned content. Returns: diff --git a/omniread/csv/scraper.pyi b/omniread/csv/scraper.pyi new file mode 100644 index 0000000..3328806 --- /dev/null +++ b/omniread/csv/scraper.pyi @@ -0,0 +1,12 @@ +from collections.abc import Mapping +from typing import Any + +from omniread.core.content import Content + +from .client import BaseCsvClient + +class CsvScraper: + def __init__(self, *, client: BaseCsvClient) -> None: ... + def fetch( + self, source: Any, *, metadata: Mapping[str, Any] | None = ... + ) -> Content: ... diff --git a/omniread/html/__init__.py b/omniread/html/__init__.py index bd9dae0..6922648 100644 --- a/omniread/html/__init__.py +++ b/omniread/html/__init__.py @@ -30,7 +30,6 @@ use this package only when HTML-specific behavior is required. --- """ - from .scraper import HTMLScraper from .parser import HTMLParser diff --git a/omniread/html/__init__.pyi b/omniread/html/__init__.pyi index e52c56d..5bdc4fa 100644 --- a/omniread/html/__init__.pyi +++ b/omniread/html/__init__.pyi @@ -1,4 +1,4 @@ -from .scraper import HTMLScraper from .parser import HTMLParser +from .scraper import HTMLScraper __all__ = ["HTMLScraper", "HTMLParser"] diff --git a/omniread/html/parser.py b/omniread/html/parser.py index de9b49b..101a1d9 100644 --- a/omniread/html/parser.py +++ b/omniread/html/parser.py @@ -16,12 +16,12 @@ Concrete parsers must subclass `HTMLParser` and implement the `parse()` method to return a structured representation appropriate for their use case. """ -from typing import Any, Generic, TypeVar, Optional from abc import abstractmethod +from typing import Any, Generic, TypeVar from bs4 import BeautifulSoup, Tag -from omniread.core.content import ContentType, Content +from omniread.core.content import Content, ContentType from omniread.core.parser import BaseParser T = TypeVar("T") @@ -117,7 +117,7 @@ class HTMLParser(BaseParser[T], Generic[T]): return div.get_text(separator=separator, strip=True) @staticmethod - def parse_link(a: Tag) -> Optional[str]: + def parse_link(a: Tag) -> str | None: """ Extract the hyperlink reference from an `` element. @@ -126,7 +126,7 @@ class HTMLParser(BaseParser[T], Generic[T]): BeautifulSoup tag representing an anchor. Returns: - Optional[str]: + str | None: The value of the `href` attribute, or None if absent. """ return a.get("href") @@ -146,10 +146,7 @@ class HTMLParser(BaseParser[T], Generic[T]): """ rows: list[list[str]] = [] for tr in table.find_all("tr"): - cells = [ - cell.get_text(strip=True) - for cell in tr.find_all(["td", "th"]) - ] + cells = [cell.get_text(strip=True) for cell in tr.find_all(["td", "th"])] if cells: rows.append(cells) return rows diff --git a/omniread/html/parser.pyi b/omniread/html/parser.pyi index 78d8fad..78a7c2d 100644 --- a/omniread/html/parser.pyi +++ b/omniread/html/parser.pyi @@ -1,6 +1,8 @@ -from typing import Any, Generic, TypeVar, Optional, list, dict -from bs4 import BeautifulSoup, Tag -from omniread.core.content import ContentType, Content +from typing import Any, Generic, TypeVar + +from bs4 import Tag + +from omniread.core.content import Content, ContentType from omniread.core.parser import BaseParser T = TypeVar("T") @@ -12,7 +14,7 @@ class HTMLParser(BaseParser[T], Generic[T]): @staticmethod def parse_div(div: Tag, *, separator: str = ...) -> str: ... @staticmethod - def parse_link(a: Tag) -> Optional[str]: ... + def parse_link(a: Tag) -> str | None: ... @staticmethod def parse_table(table: Tag) -> list[list[str]]: ... def parse_meta(self) -> dict[str, Any]: ... diff --git a/omniread/html/scraper.py b/omniread/html/scraper.py index 8d13635..5c38367 100644 --- a/omniread/html/scraper.py +++ b/omniread/html/scraper.py @@ -20,8 +20,10 @@ This scraper is not responsible for: - Managing crawl policies or rate limiting """ +from collections.abc import Mapping +from typing import Any + import httpx -from typing import Any, Mapping, Optional from omniread.core.content import Content, ContentType from omniread.core.scraper import BaseScraper @@ -51,7 +53,7 @@ class HTMLScraper(BaseScraper): *, client: httpx.Client | None = None, timeout: float = 15.0, - headers: Optional[Mapping[str, str]] = None, + headers: Mapping[str, str] | None = None, follow_redirects: bool = True, ): """ @@ -62,7 +64,7 @@ class HTMLScraper(BaseScraper): Optional pre-configured `httpx.Client`. If omitted, a client is created internally. timeout (float, optional): Request timeout in seconds. - headers (Optional[Mapping[str, str]], optional): + headers (Mapping[str, str] | None, optional): Optional default HTTP headers. follow_redirects (bool, optional): Whether to follow HTTP redirects. @@ -97,15 +99,13 @@ class HTMLScraper(BaseScraper): base_ct = raw_ct.split(";", 1)[0].strip().lower() if base_ct != self.content_type.value: - raise ValueError( - f"Expected HTML content, got '{raw_ct}'" - ) + raise ValueError(f"Expected HTML content, got '{raw_ct}'") def fetch( self, source: str, *, - metadata: Optional[Mapping[str, Any]] = None, + metadata: Mapping[str, Any] | None = None, ) -> Content: """ Fetch an HTML document from the given source. @@ -113,7 +113,7 @@ class HTMLScraper(BaseScraper): Args: source (str): URL of the HTML document. - metadata (Optional[Mapping[str, Any]], optional): + metadata (Mapping[str, Any] | None, optional): Optional metadata to be merged into the returned content. Returns: diff --git a/omniread/html/scraper.pyi b/omniread/html/scraper.pyi index 2249bca..43838c7 100644 --- a/omniread/html/scraper.pyi +++ b/omniread/html/scraper.pyi @@ -1,10 +1,22 @@ +from collections.abc import Mapping +from typing import Any + import httpx -from typing import Any, Mapping, Optional + from omniread.core.content import Content, ContentType from omniread.core.scraper import BaseScraper class HTMLScraper(BaseScraper): content_type: ContentType - def __init__(self, *, client: Optional[httpx.Client] = ..., timeout: float = ..., headers: Optional[Mapping[str, str]] = ..., follow_redirects: bool = ...) -> None: ... + def __init__( + self, + *, + client: httpx.Client | None = ..., + timeout: float = ..., + headers: Mapping[str, str] | None = ..., + follow_redirects: bool = ..., + ) -> None: ... def validate_content_type(self, response: httpx.Response) -> None: ... - def fetch(self, source: str, *, metadata: Optional[Mapping[str, Any]] = ...) -> Content: ... + def fetch( + self, source: str, *, metadata: Mapping[str, Any] | None = ... + ) -> Content: ... diff --git a/omniread/pdf/__init__.pyi b/omniread/pdf/__init__.pyi index bfd206e..7687f77 100644 --- a/omniread/pdf/__init__.pyi +++ b/omniread/pdf/__init__.pyi @@ -1,5 +1,5 @@ from .client import FileSystemPDFClient -from .scraper import PDFScraper from .parser import PDFParser +from .scraper import PDFScraper __all__ = ["FileSystemPDFClient", "PDFScraper", "PDFParser"] diff --git a/omniread/pdf/client.py b/omniread/pdf/client.py index ee93c64..dd64834 100644 --- a/omniread/pdf/client.py +++ b/omniread/pdf/client.py @@ -17,9 +17,9 @@ Typical backing stores include: - Network file systems """ -from typing import Any from abc import ABC, abstractmethod from pathlib import Path +from typing import Any class BasePDFClient(ABC): diff --git a/omniread/pdf/parser.py b/omniread/pdf/parser.py index 2e010a8..e2c2430 100644 --- a/omniread/pdf/parser.py +++ b/omniread/pdf/parser.py @@ -10,8 +10,8 @@ PDF parsers are responsible for interpreting binary PDF data and producing structured representations suitable for downstream consumption. """ -from typing import Generic, TypeVar from abc import abstractmethod +from typing import Generic, TypeVar from omniread.core.content import ContentType from omniread.core.parser import BaseParser diff --git a/omniread/pdf/parser.pyi b/omniread/pdf/parser.pyi index 79439f7..c81d625 100644 --- a/omniread/pdf/parser.pyi +++ b/omniread/pdf/parser.pyi @@ -1,5 +1,6 @@ from abc import abstractmethod from typing import Generic, TypeVar + from omniread.core.content import ContentType from omniread.core.parser import BaseParser diff --git a/omniread/pdf/scraper.py b/omniread/pdf/scraper.py index 4726580..a217d17 100644 --- a/omniread/pdf/scraper.py +++ b/omniread/pdf/scraper.py @@ -10,7 +10,8 @@ The scraper implements the core `BaseScraper` contract while delegating all storage and access concerns to a `BasePDFClient` implementation. """ -from typing import Any, Mapping, Optional +from collections.abc import Mapping +from typing import Any from omniread.core.content import Content, ContentType from omniread.core.scraper import BaseScraper @@ -48,7 +49,7 @@ class PDFScraper(BaseScraper): self, source: Any, *, - metadata: Optional[Mapping[str, Any]] = None, + metadata: Mapping[str, Any] | None = None, ) -> Content: """ Fetch a PDF document from the given source. @@ -56,7 +57,7 @@ class PDFScraper(BaseScraper): Args: source (Any): Identifier of the PDF source as understood by the configured PDF client. - metadata (Optional[Mapping[str, Any]], optional): + metadata (Mapping[str, Any] | None, optional): Optional metadata to attach to the returned content. Returns: diff --git a/omniread/pdf/scraper.pyi b/omniread/pdf/scraper.pyi index cb4ba75..af3fbd5 100644 --- a/omniread/pdf/scraper.pyi +++ b/omniread/pdf/scraper.pyi @@ -1,8 +1,13 @@ -from typing import Any, Mapping, Optional -from omniread.core.content import Content, ContentType +from collections.abc import Mapping +from typing import Any + +from omniread.core.content import Content from omniread.core.scraper import BaseScraper + from .client import BasePDFClient class PDFScraper(BaseScraper): def __init__(self, *, client: BasePDFClient) -> None: ... - def fetch(self, source: Any, *, metadata: Optional[Mapping[str, Any]] = ...) -> Content: ... + def fetch( + self, source: Any, *, metadata: Mapping[str, Any] | None = ... + ) -> Content: ... diff --git a/omniread/py.typed b/omniread/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/omniread/xlsx/__init__.pyi b/omniread/xlsx/__init__.pyi new file mode 100644 index 0000000..e9b5c3b --- /dev/null +++ b/omniread/xlsx/__init__.pyi @@ -0,0 +1,12 @@ +from .client import BaseXlsxClient, FileSystemXlsxClient +from .parser import XlsxParser +from .parser_base import XlsxParserBase +from .scraper import XlsxScraper + +__all__ = [ + "BaseXlsxClient", + "FileSystemXlsxClient", + "XlsxScraper", + "XlsxParser", + "XlsxParserBase", +] diff --git a/omniread/xlsx/client.py b/omniread/xlsx/client.py index b7c9409..a3c67e3 100644 --- a/omniread/xlsx/client.py +++ b/omniread/xlsx/client.py @@ -17,9 +17,9 @@ Typical backing stores include: - Network file systems """ -from typing import Any from abc import ABC, abstractmethod from pathlib import Path +from typing import Any class BaseXlsxClient(ABC): diff --git a/omniread/xlsx/client.pyi b/omniread/xlsx/client.pyi new file mode 100644 index 0000000..48eab7c --- /dev/null +++ b/omniread/xlsx/client.pyi @@ -0,0 +1,10 @@ +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Any + +class BaseXlsxClient(ABC): + @abstractmethod + def fetch(self, source: Any) -> bytes: ... + +class FileSystemXlsxClient(BaseXlsxClient): + def fetch(self, path: Path) -> bytes: ... diff --git a/omniread/xlsx/parser.py b/omniread/xlsx/parser.py index bb70288..c8f1188 100644 --- a/omniread/xlsx/parser.py +++ b/omniread/xlsx/parser.py @@ -14,15 +14,16 @@ detection or column interpretation beyond basic cell normalization. import datetime from io import BytesIO -from typing import List, Optional, Union +from typing import Any import openpyxl from omniread.core.content import Content + from .parser_base import XlsxParserBase -class XlsxParser(XlsxParserBase): +class XlsxParser(XlsxParserBase[list[list[str]]]): """ Generic xlsx parser producing string rows from a worksheet. @@ -41,7 +42,9 @@ class XlsxParser(XlsxParserBase): locale-specific formatting must convert on their side. """ - def __init__(self, content: Content, *, data_only: bool = True, read_only: bool = True): + def __init__( + self, content: Content, *, data_only: bool = True, read_only: bool = True + ): """ Initialize the parser. @@ -57,7 +60,7 @@ class XlsxParser(XlsxParserBase): super().__init__(content) self._data_only = data_only self._read_only = read_only - self._workbook: Optional[openpyxl.Workbook] = None + self._workbook: openpyxl.Workbook | None = None @property def workbook(self) -> openpyxl.Workbook: @@ -73,39 +76,39 @@ class XlsxParser(XlsxParserBase): return self._workbook @property - def sheet_names(self) -> List[str]: + def sheet_names(self) -> list[str]: """ Names of all worksheets contained in the workbook. """ return list(self.workbook.sheetnames) - def parse(self) -> List[List[str]]: + def parse(self) -> list[list[str]]: """ Parse the first worksheet into normalized string rows. Returns: - List[List[str]]: + list[list[str]]: Rows of the default (first) worksheet. """ return self.rows() def rows( self, - sheet: Optional[Union[int, str]] = None, + sheet: int | str | None = None, *, skip_empty: bool = True, - ) -> List[List[str]]: + ) -> list[list[str]]: """ Extract normalized string rows from a worksheet. Args: - sheet (Optional[Union[int, str]]): + sheet (int | str | None): Worksheet index or title; defaults to the first worksheet. skip_empty (bool): When True (default), rows whose cells are all blank are omitted. Returns: - List[List[str]]: + list[list[str]]: Normalized rows; trailing blank cells are trimmed per row. Raises: @@ -113,7 +116,7 @@ class XlsxParser(XlsxParserBase): If the requested sheet does not exist. """ ws = self._resolve_sheet(sheet) - out: List[List[str]] = [] + out: list[list[str]] = [] for row in ws.iter_rows(values_only=True): cells = [self._cell_str(v) for v in row] while cells and not cells[-1].strip(): @@ -123,7 +126,7 @@ class XlsxParser(XlsxParserBase): out.append(cells) return out - def _resolve_sheet(self, sheet: Optional[Union[int, str]]): + def _resolve_sheet(self, sheet: int | str | None) -> Any: names = self.workbook.sheetnames if not names: raise ValueError("Workbook contains no worksheets") @@ -138,7 +141,7 @@ class XlsxParser(XlsxParserBase): return self.workbook[names[index]] @staticmethod - def _cell_str(value) -> str: + def _cell_str(value: Any) -> str: if value is None: return "" if isinstance(value, (datetime.datetime, datetime.date)): diff --git a/omniread/xlsx/parser.pyi b/omniread/xlsx/parser.pyi new file mode 100644 index 0000000..33f37aa --- /dev/null +++ b/omniread/xlsx/parser.pyi @@ -0,0 +1,18 @@ +import openpyxl + +from omniread.core.content import Content + +from .parser_base import XlsxParserBase + +class XlsxParser(XlsxParserBase[list[list[str]]]): + def __init__( + self, content: Content, *, data_only: bool = True, read_only: bool = True + ) -> None: ... + @property + def workbook(self) -> openpyxl.Workbook: ... + @property + def sheet_names(self) -> list[str]: ... + def parse(self) -> list[list[str]]: ... + def rows( + self, sheet: int | str | None = None, *, skip_empty: bool = True + ) -> list[list[str]]: ... diff --git a/omniread/xlsx/parser_base.py b/omniread/xlsx/parser_base.py index 9974ef6..06acfbf 100644 --- a/omniread/xlsx/parser_base.py +++ b/omniread/xlsx/parser_base.py @@ -8,8 +8,8 @@ format-agnostic `BaseParser` with constraints appropriate for Office Open XML spreadsheet content. """ -from typing import Generic, TypeVar from abc import abstractmethod +from typing import Generic, TypeVar from omniread.core.content import ContentType from omniread.core.parser import BaseParser diff --git a/omniread/xlsx/parser_base.pyi b/omniread/xlsx/parser_base.pyi new file mode 100644 index 0000000..561df4e --- /dev/null +++ b/omniread/xlsx/parser_base.pyi @@ -0,0 +1,13 @@ +from abc import abstractmethod +from typing import Generic, TypeVar + +from omniread.core.content import ContentType +from omniread.core.parser import BaseParser + +T = TypeVar("T") + +class XlsxParserBase(BaseParser[T], Generic[T]): + supported_types: set[ContentType] + + @abstractmethod + def parse(self) -> T: ... diff --git a/omniread/xlsx/scraper.py b/omniread/xlsx/scraper.py index 00202ad..d4319f9 100644 --- a/omniread/xlsx/scraper.py +++ b/omniread/xlsx/scraper.py @@ -10,9 +10,11 @@ The scraper does not interpret or parse the acquired bytes; it wraps them in the canonical `Content` model. """ -from typing import Any, Mapping, Optional +from collections.abc import Mapping +from typing import Any from omniread.core.content import Content, ContentType + from .client import BaseXlsxClient @@ -47,7 +49,7 @@ class XlsxScraper: self, source: Any, *, - metadata: Optional[Mapping[str, Any]] = None, + metadata: Mapping[str, Any] | None = None, ) -> Content: """ Fetch an xlsx document from the given source. @@ -56,7 +58,7 @@ class XlsxScraper: source (Any): Identifier of the spreadsheet source as understood by the configured client. - metadata (Optional[Mapping[str, Any]], optional): + metadata (Mapping[str, Any] | None, optional): Optional metadata to attach to the returned content. Returns: diff --git a/omniread/xlsx/scraper.pyi b/omniread/xlsx/scraper.pyi new file mode 100644 index 0000000..df74b98 --- /dev/null +++ b/omniread/xlsx/scraper.pyi @@ -0,0 +1,12 @@ +from collections.abc import Mapping +from typing import Any + +from omniread.core.content import Content + +from .client import BaseXlsxClient + +class XlsxScraper: + def __init__(self, *, client: BaseXlsxClient) -> None: ... + def fetch( + self, source: Any, *, metadata: Mapping[str, Any] | None = ... + ) -> Content: ... diff --git a/pyproject.toml b/pyproject.toml index fb78aff..27f4acf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,14 @@ [build-system] -requires = ["setuptools>=65.0", "wheel"] +requires = ["setuptools>=68", "wheel"] build-backend = "setuptools.build_meta" + [project] name = "omniread" -version = "0.0.1" +version = "0.0.3" description = "Composable content ingestion framework with pluggable scrapers and parsers for HTML, PDF, and structured data" readme = "README.md" -requires-python = ">=3.9" +requires-python = ">=3.10" license = { text = "MIT" } authors = [ @@ -17,6 +18,7 @@ maintainers = [ { name = "Aetos Skia", email = "dev@aetoskia.com" } ] + keywords = [ "scraping", "parsing", @@ -33,39 +35,49 @@ classifiers = [ "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", - "Topic :: Security", "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Text Processing", "Typing :: Typed", ] + dependencies = [ "requests>=2.31.0", "beautifulsoup4>=4.12.0", -# "lxml>=5.0.0", "pypdf>=4.0.0", "openpyxl>=3.1.0", ] [project.optional-dependencies] dev = [ - "pytest>=7.4.0", + "pytest>=8.0.0", "pytest-asyncio>=0.21.0", "pytest-cov>=4.1.0", "black>=23.0.0", - "ruff>=0.1.0", - "mypy>=1.5.0", + "ruff>=0.3.0", + "mypy>=1.8.0", + "build>=1.0.0", + "twine>=4.0.0", "pre-commit>=3.4.0", + "doc-forge[mcp,mkdocs]>=0.0.6", +] + +docs = [ + "mkdocs>=1.5.0", + "mkdocs-material>=9.5.0", + "mkdocstrings[python]>=0.24.0", ] all = [ - "omniread[dev,fastapi]", + "omniread[dev,docs]", ] + [project.urls] Homepage = "https://git.aetoskia.com/aetos/omniread" Documentation = "https://git.aetoskia.com/aetos/omniread#readme" @@ -73,11 +85,13 @@ Repository = "https://git.aetoskia.com/aetos/omniread.git" Issues = "https://git.aetoskia.com/aetos/omniread/issues" Versions = "https://git.aetoskia.com/aetos/omniread/tags" + [tool.setuptools] packages = { find = { include = ["omniread*"] } } [tool.setuptools.package-data] -omniread = ["py.typed"] +omniread = ['py.typed', '\.pyi?$'] + [tool.pytest.ini_options] testpaths = ["tests"] @@ -94,9 +108,10 @@ addopts = [ "--cov-report=xml", ] + [tool.black] line-length = 88 -target-version = ["py39", "py310", "py311", "py312", "py313"] +target-version = ["py310", "py311", "py312", "py313"] include = '\.pyi?$' extend-exclude = ''' /( @@ -111,9 +126,12 @@ extend-exclude = ''' )/ ''' + [tool.ruff] line-length = 88 -target-version = "py39" +target-version = "py310" + +[tool.ruff.lint] select = [ "E", "W", @@ -129,31 +147,38 @@ ignore = [ "C901", ] -[tool.ruff.per-file-ignores] -"__init__.py" = ["F401"] +[tool.ruff.lint.per-file-ignores] +"__init__.py" = ["F401", "I001"] +"tests/*" = ["B008"] + [tool.mypy] -python_version = "3.9" -warn_return_any = true -warn_unused_configs = true -disallow_untyped_defs = true -disallow_incomplete_defs = true -check_untyped_defs = true -disallow_untyped_decorators = false -no_implicit_optional = true -warn_redundant_casts = true -warn_unused_ignores = true -warn_no_return = true -follow_imports = "normal" -strict_optional = true +python_version = "3.10" +strict = true +exclude = [ + "tests/", +] +files = ["omniread"] [[tool.mypy.overrides]] module = [ - "jose.*", - "httpx.*", + "requests.*", + "bs4.*", + "lxml.*", + "pypdf.*", + "openpyxl.*", ] ignore_missing_imports = true + +[tool.pydoclint] +style = "google" +exclude = '\.pyi$|tests/' +allow-init-docstring = true +skip-checking-raises = true +check-class-attributes = false + + [tool.coverage.run] source = ["omniread"] omit = [ @@ -170,4 +195,4 @@ exclude_lines = [ "raise NotImplementedError", "if TYPE_CHECKING:", "@abstractmethod", -] +] \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 309c278..2a35ef2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,22 +1,20 @@ import json -import pytest -import httpx from pathlib import Path -from jinja2 import Environment, BaseLoader + +import httpx +import pytest +from jinja2 import BaseLoader, Environment from omniread import ( # core ContentType, - - # html - HTMLScraper, - # pdf FileSystemPDFClient, + # html + HTMLScraper, PDFScraper, ) - MOCK_HTML_DIR = Path(__file__).parent / "mocks" / "html" MOCK_PDF_DIR = Path(__file__).parent / "mocks" / "pdf" @@ -40,7 +38,7 @@ def mock_transport(request: httpx.Request) -> httpx.Response: httpx MockTransport handler. """ path = request.url.path - if path not in ['/simple', '/table']: + if path not in ["/simple", "/table"]: return httpx.Response( status_code=404, content=b"Not Found", diff --git a/tests/test_html_simple.py b/tests/test_html_simple.py index cd32f0c..9ec751a 100644 --- a/tests/test_html_simple.py +++ b/tests/test_html_simple.py @@ -1,22 +1,19 @@ -from typing import Optional - -from pydantic import BaseModel from bs4 import Tag +from pydantic import BaseModel from omniread import ( # core Content, - # html HTMLParser, ) class ParsedSimpleHTML(BaseModel): - title: Optional[str] - description: Optional[str] - content: Optional[str] - link: Optional[str] + title: str | None + description: str | None + content: str | None + link: str | None class SimpleHTMLParser(HTMLParser[ParsedSimpleHTML]): diff --git a/tests/test_html_table.py b/tests/test_html_table.py index dd4a133..b6ad656 100644 --- a/tests/test_html_table.py +++ b/tests/test_html_table.py @@ -1,18 +1,15 @@ -from typing import Optional - from pydantic import BaseModel from omniread import ( # core Content, - # html HTMLParser, ) class ParsedTableHTML(BaseModel): - title: Optional[str] + title: str | None table: list[list[str]] diff --git a/tests/test_pdf_simple.py b/tests/test_pdf_simple.py index b90f791..cd6f82e 100644 --- a/tests/test_pdf_simple.py +++ b/tests/test_pdf_simple.py @@ -1,14 +1,15 @@ from typing import Literal + from pydantic import BaseModel from omniread import ( # core Content, - # pdf PDFParser, ) + class ParsedPDF(BaseModel): size_bytes: int magic: Literal[b"%PDF"] diff --git a/tests/test_xlsx_simple.py b/tests/test_xlsx_simple.py index 1390b19..4acaa42 100644 --- a/tests/test_xlsx_simple.py +++ b/tests/test_xlsx_simple.py @@ -8,7 +8,6 @@ from omniread import ( # core Content, ContentType, - # xlsx FileSystemXlsxClient, XlsxParser, @@ -22,8 +21,28 @@ def _fixture_xlsx_bytes() -> bytes: ws.title = "Statement" ws.append(["DETAILED STATEMENT", None, None, ""]) ws.append([]) - ws.append(["S No", "Value Date", "Txn Date", "Remarks", "Withdrawal", "Deposit", "Balance"]) - ws.append([1, "01/06/2026", "01/06/2026", "UPI/NPCI BHIM/bhimcashback@h/B", 0.0, 10.0, 250285.24]) + ws.append( + [ + "S No", + "Value Date", + "Txn Date", + "Remarks", + "Withdrawal", + "Deposit", + "Balance", + ] + ) + ws.append( + [ + 1, + "01/06/2026", + "01/06/2026", + "UPI/NPCI BHIM/bhimcashback@h/B", + 0.0, + 10.0, + 250285.24, + ] + ) row = 4 ws.cell(row=row, column=5).value = 100.0 # ensure float rendering path buf = BytesIO() @@ -68,7 +87,9 @@ def test_client_directory_raises(tmp_path): def test_parser_rows_skip_empty_and_trim(xlsx_bytes): - parser = XlsxParser(Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX)) + parser = XlsxParser( + Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX) + ) rows = parser.rows() @@ -76,14 +97,24 @@ def test_parser_rows_skip_empty_and_trim(xlsx_bytes): assert len(rows) == 3 assert rows[0] == ["DETAILED STATEMENT"] header = rows[1] - assert header[:7] == ["S No", "Value Date", "Txn Date", "Remarks", "Withdrawal", "Deposit", "Balance"] + assert header[:7] == [ + "S No", + "Value Date", + "Txn Date", + "Remarks", + "Withdrawal", + "Deposit", + "Balance", + ] data = rows[2] assert data[3] == "UPI/NPCI BHIM/bhimcashback@h/B" assert data[6] == "250285.24" def test_parser_keep_empty_rows(xlsx_bytes): - parser = XlsxParser(Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX)) + parser = XlsxParser( + Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX) + ) rows = parser.rows(skip_empty=False) @@ -91,7 +122,9 @@ def test_parser_keep_empty_rows(xlsx_bytes): def test_parser_sheet_selection_and_names(xlsx_bytes): - parser = XlsxParser(Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX)) + parser = XlsxParser( + Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX) + ) assert parser.sheet_names == ["Statement"] assert parser.rows(sheet="Statement") == parser.rows(sheet=0) @@ -108,11 +141,15 @@ def test_parser_date_cell_isoformat(): buf = BytesIO() wb.save(buf) - parser = XlsxParser(Content(raw=buf.getvalue(), source="mem", content_type=ContentType.XLSX)) + parser = XlsxParser( + Content(raw=buf.getvalue(), source="mem", content_type=ContentType.XLSX) + ) # openpyxl surfaces date cells as datetime; ISO rendering keeps determinism assert parser.parse()[0][0] == "2026-06-01T00:00:00" def test_parser_rejects_non_xlsx_content(): with pytest.raises(ValueError, match="does not support"): - XlsxParser(Content(raw=b"", source="mem", content_type=ContentType.HTML)) + XlsxParser( + Content(raw=b"", source="mem", content_type=ContentType.HTML) + )