From c3f3c9e57bcb2cd09cb7b115f175ac16f804de2a Mon Sep 17 00:00:00 2001 From: Vishesh 'ironeagle' Bangotra Date: Mon, 24 Aug 2026 20:19:33 +0000 Subject: [PATCH] xlsx-and-csv-read (#3) Reviewed-on: https://git.aetoskia.com/aetos/omniread/pulls/3 Co-authored-by: Vishesh 'ironeagle' Bangotra Co-committed-by: Vishesh 'ironeagle' Bangotra --- README.md | 108 +++++++ mcp_docs/modules/omniread.core.content.json | 18 +- mcp_docs/modules/omniread.core.json | 92 +++--- mcp_docs/modules/omniread.core.parser.json | 28 +- mcp_docs/modules/omniread.core.scraper.json | 20 +- mcp_docs/modules/omniread.html.json | 96 +++--- mcp_docs/modules/omniread.html.parser.json | 46 +-- mcp_docs/modules/omniread.html.scraper.json | 30 +- mcp_docs/modules/omniread.json | 314 ++++++++++---------- mcp_docs/modules/omniread.pdf.client.json | 18 +- mcp_docs/modules/omniread.pdf.json | 82 ++--- mcp_docs/modules/omniread.pdf.parser.json | 20 +- mcp_docs/modules/omniread.pdf.scraper.json | 30 +- mkdocs.yml | 33 +- omniread/__init__.py | 181 ++++++----- omniread/core/__init__.py | 18 +- omniread/core/content.py | 52 +++- omniread/core/parser.py | 53 ++-- omniread/core/scraper.py | 55 ++-- omniread/csv/__init__.py | 26 ++ omniread/csv/client.py | 97 ++++++ omniread/csv/parser.py | 102 +++++++ omniread/csv/parser_base.py | 55 ++++ omniread/csv/scraper.py | 79 +++++ omniread/html/__init__.py | 25 +- omniread/html/parser.py | 96 +++--- omniread/html/scraper.py | 67 +++-- omniread/pdf/__init__.py | 19 +- omniread/pdf/client.py | 48 ++- omniread/pdf/parser.py | 35 ++- omniread/pdf/scraper.py | 38 ++- omniread/xlsx/__init__.py | 27 ++ omniread/xlsx/client.py | 97 ++++++ omniread/xlsx/parser.py | 146 +++++++++ omniread/xlsx/parser_base.py | 55 ++++ omniread/xlsx/scraper.py | 78 +++++ pyproject.toml | 1 + tests/test_xlsx_simple.py | 118 ++++++++ 38 files changed, 1847 insertions(+), 656 deletions(-) create mode 100644 README.md create mode 100644 omniread/csv/__init__.py create mode 100644 omniread/csv/client.py create mode 100644 omniread/csv/parser.py create mode 100644 omniread/csv/parser_base.py create mode 100644 omniread/csv/scraper.py create mode 100644 omniread/xlsx/__init__.py create mode 100644 omniread/xlsx/client.py create mode 100644 omniread/xlsx/parser.py create mode 100644 omniread/xlsx/parser_base.py create mode 100644 omniread/xlsx/scraper.py create mode 100644 tests/test_xlsx_simple.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..0a66f72 --- /dev/null +++ b/README.md @@ -0,0 +1,108 @@ +# omniread + +# Summary + +`OmniRead` — format-agnostic content acquisition and parsing framework. + +`OmniRead` provides a **cleanly layered architecture** for fetching, parsing, +and normalizing content from heterogeneous sources such as HTML documents +and PDF files. + +The library is structured around three core concepts: + +1. **`Content`**: A canonical, format-agnostic container representing raw content + bytes and minimal contextual metadata. +2. **`Scrapers`**: Components responsible for *acquiring* raw content from a + source (HTTP, filesystem, object storage, etc.). `Scrapers` never interpret + content. +3. **`Parsers`**: Components responsible for *interpreting* acquired content and + converting it into structured, typed representations. + +`OmniRead` deliberately separates these responsibilities to ensure: + +- Clear boundaries between IO and interpretation. +- Replaceable implementations per format. +- Predictable, testable behavior. + +# Installation + +Install `OmniRead` using pip: + +```bash +pip install omniread +``` + +Install OmniRead using Poetry: +```bash +poetry add omniread +``` + +--- + +## Quick start + +Example: + HTML example: + ```python + from omniread import HTMLScraper, HTMLParser + + scraper = HTMLScraper() + content = scraper.fetch("https://example.com") + + class TitleParser(HTMLParser[str]): + def parse(self) -> str: + return self._soup.title.string + + parser = TitleParser(content) + title = parser.parse() + ``` + + PDF example: + ```python + from omniread import FileSystemPDFClient, PDFScraper, PDFParser + from pathlib import Path + + client = FileSystemPDFClient() + scraper = PDFScraper(client=client) + content = scraper.fetch(Path("document.pdf")) + + class TextPDFParser(PDFParser[str]): + def parse(self) -> str: + # implement PDF text extraction + ... + + parser = TextPDFParser(content) + result = parser.parse() + ``` + +--- + +# Public API + +This module re-exports the **recommended public entry points** of OmniRead. +Consumers are encouraged to import from this namespace rather than from +format-specific submodules directly, unless advanced customization is +required. + +- `Content`: Canonical content model. +- `ContentType`: Supported media types. +- `HTMLScraper`: HTTP-based HTML acquisition. +- `HTMLParser`: Base parser for HTML DOM interpretation. +- `FileSystemPDFClient`: Local filesystem PDF access. +- `PDFScraper`: PDF-specific content acquisition. +- `PDFParser`: Base parser for PDF binary interpretation. + +--- + +# Core Philosophy + +`OmniRead` is designed as a **decoupled content engine**: + +1. **Separation of Concerns**: Scrapers *fetch*, Parsers *interpret*. Neither + knows about the other. +2. **Normalized Exchange**: All components communicate via the `Content` model, + ensuring a consistent contract. +3. **Format Agnosticism**: The core logic is independent of whether the input + is HTML, PDF, or JSON. + +--- diff --git a/mcp_docs/modules/omniread.core.content.json b/mcp_docs/modules/omniread.core.content.json index 811112d..54a56c3 100644 --- a/mcp_docs/modules/omniread.core.content.json +++ b/mcp_docs/modules/omniread.core.content.json @@ -2,7 +2,7 @@ "module": "omniread.core.content", "content": { "path": "omniread.core.content", - "docstring": "Canonical content models for OmniRead.\n\nThis module defines the **format-agnostic content representation** used across\nall parsers and scrapers in OmniRead.\n\nThe models defined here represent *what* was extracted, not *how* it was\nretrieved or parsed. Format-specific behavior and metadata must not alter\nthe semantic meaning of these models.", + "docstring": "# Summary\n\nCanonical content models for OmniRead.\n\nThis module defines the **format-agnostic content representation** used across\nall parsers and scrapers in OmniRead.\n\nThe models defined here represent *what* was extracted, not *how* it was\nretrieved or parsed. Format-specific behavior and metadata must not alter\nthe semantic meaning of these models.", "objects": { "Enum": { "name": "Enum", @@ -43,8 +43,8 @@ "name": "ContentType", "kind": "class", "path": "omniread.core.content.ContentType", - "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nThis enum represents the declared or inferred media type of the content\nsource. It is primarily used for routing content to the appropriate\nparser or downstream consumer.", + "signature": "", + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", @@ -80,36 +80,36 @@ "name": "Content", "kind": "class", "path": "omniread.core.content.Content", - "signature": "", - "docstring": "Normalized representation of extracted content.\n\nA `Content` instance represents a raw content payload along with minimal\ncontextual metadata describing its origin and type.\n\nThis class is the **primary exchange format** between:\n- Scrapers\n- Parsers\n- Downstream consumers\n\nAttributes:\n raw: Raw content bytes as retrieved from the source.\n source: Identifier of the content origin (URL, file path, or logical name).\n content_type: Optional MIME type of the content, if known.\n metadata: Optional, implementation-defined metadata associated with\n the content (e.g., headers, encoding hints, extraction notes).", + "signature": "", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.core.content.Content.raw", "signature": null, - "docstring": null + "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.core.content.Content.source", "signature": null, - "docstring": null + "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.core.content.Content.content_type", "signature": null, - "docstring": null + "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.core.content.Content.metadata", "signature": null, - "docstring": null + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } } diff --git a/mcp_docs/modules/omniread.core.json b/mcp_docs/modules/omniread.core.json index 068f792..2cd7094 100644 --- a/mcp_docs/modules/omniread.core.json +++ b/mcp_docs/modules/omniread.core.json @@ -2,42 +2,42 @@ "module": "omniread.core", "content": { "path": "omniread.core", - "docstring": "Core domain contracts for OmniRead.\n\nThis package defines the **format-agnostic domain layer** of OmniRead.\nIt exposes canonical content models and abstract interfaces that are\nimplemented by format-specific modules (HTML, PDF, etc.).\n\nPublic exports from this package are considered **stable contracts** and\nare safe for downstream consumers to depend on.\n\nSubmodules:\n- content: Canonical content models and enums\n- parser: Abstract parsing contracts\n- scraper: Abstract scraping contracts\n\nFormat-specific behavior must not be introduced at this layer.", + "docstring": "# Summary\n\nCore domain contracts for OmniRead.\n\nThis package defines the **format-agnostic domain layer** of OmniRead.\nIt exposes canonical content models and abstract interfaces that are\nimplemented by format-specific modules (HTML, PDF, etc.).\n\nPublic exports from this package are considered **stable contracts** and\nare safe for downstream consumers to depend on.\n\nSubmodules:\n\n- `content`: Canonical content models and enums.\n- `parser`: Abstract parsing contracts.\n- `scraper`: Abstract scraping contracts.\n\nFormat-specific behavior must not be introduced at this layer.\n\n---\n\n# Public API\n\n- `Content`\n- `ContentType`\n\n---", "objects": { "Content": { "name": "Content", "kind": "class", "path": "omniread.core.Content", "signature": "", - "docstring": "Normalized representation of extracted content.\n\nA `Content` instance represents a raw content payload along with minimal\ncontextual metadata describing its origin and type.\n\nThis class is the **primary exchange format** between:\n- Scrapers\n- Parsers\n- Downstream consumers\n\nAttributes:\n raw: Raw content bytes as retrieved from the source.\n source: Identifier of the content origin (URL, file path, or logical name).\n content_type: Optional MIME type of the content, if known.\n metadata: Optional, implementation-defined metadata associated with\n the content (e.g., headers, encoding hints, extraction notes).", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.core.Content.raw", "signature": "", - "docstring": null + "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.core.Content.source", "signature": "", - "docstring": null + "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.core.Content.content_type", "signature": "", - "docstring": null + "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.core.Content.metadata", "signature": "", - "docstring": null + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } }, @@ -46,7 +46,7 @@ "kind": "class", "path": "omniread.core.ContentType", "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nThis enum represents the declared or inferred media type of the content\nsource. It is primarily used for routing content to the appropriate\nparser or downstream consumer.", + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", @@ -83,14 +83,14 @@ "kind": "class", "path": "omniread.core.BaseParser", "signature": "", - "docstring": "Base interface for all parsers.\n\nA parser is a self-contained object that owns the Content\nit is responsible for interpreting.\n\nImplementations must:\n- Declare supported content types via `supported_types`\n- Raise parsing-specific exceptions from `parse()`\n- Remain deterministic for a given input\n\nConsumers may rely on:\n- Early validation of content compatibility\n- Type-stable return values from `parse()`", + "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", "members": { "supported_types": { "name": "supported_types", "kind": "attribute", "path": "omniread.core.BaseParser.supported_types", "signature": "", - "docstring": "Set of content types supported by this parser.\n\nAn empty set indicates that the parser is content-type agnostic." + "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." }, "content": { "name": "content", @@ -104,14 +104,14 @@ "kind": "function", "path": "omniread.core.BaseParser.parse", "signature": "", - "docstring": "Parse the owned content into structured output.\n\nImplementations must fully consume the provided content and\nreturn a deterministic, structured output.\n\nReturns:\n Parsed, structured representation.\n\nRaises:\n Exception: Parsing-specific errors as defined by the implementation." + "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." }, "supports": { "name": "supports", "kind": "function", "path": "omniread.core.BaseParser.supports", "signature": "", - "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n True if the content type is supported; False otherwise." + "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." } } }, @@ -120,14 +120,14 @@ "kind": "class", "path": "omniread.core.BaseScraper", "signature": "", - "docstring": "Base interface for all scrapers.\n\nA scraper is responsible ONLY for fetching raw content\n(bytes) from a source. It must not interpret or parse it.\n\nA scraper is a **stateless acquisition component** that retrieves raw\ncontent from a source and returns it as a `Content` object.\n\nScrapers define *how content is obtained*, not *what the content means*.\n\nImplementations may vary in:\n- Transport mechanism (HTTP, filesystem, cloud storage)\n- Authentication strategy\n- Retry and backoff behavior\n\nImplementations must not:\n- Parse content\n- Modify content semantics\n- Couple scraping logic to a specific parser", + "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.core.BaseScraper.fetch", "signature": "", - "docstring": "Fetch raw content from the given source.\n\nImplementations must retrieve the content referenced by `source`\nand return it as raw bytes wrapped in a `Content` object.\n\nArgs:\n source: Location identifier (URL, file path, S3 URI, etc.)\n metadata: Optional hints for the scraper (headers, auth, etc.)\n\nReturns:\n Content object containing raw bytes and metadata.\n - Raw content bytes\n - Source identifier\n - Optional metadata\n\nRaises:\n Exception: Retrieval-specific errors as defined by the implementation." + "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." } } }, @@ -136,7 +136,7 @@ "kind": "module", "path": "omniread.core.content", "signature": null, - "docstring": "Canonical content models for OmniRead.\n\nThis module defines the **format-agnostic content representation** used across\nall parsers and scrapers in OmniRead.\n\nThe models defined here represent *what* was extracted, not *how* it was\nretrieved or parsed. Format-specific behavior and metadata must not alter\nthe semantic meaning of these models.", + "docstring": "# Summary\n\nCanonical content models for OmniRead.\n\nThis module defines the **format-agnostic content representation** used across\nall parsers and scrapers in OmniRead.\n\nThe models defined here represent *what* was extracted, not *how* it was\nretrieved or parsed. Format-specific behavior and metadata must not alter\nthe semantic meaning of these models.", "members": { "Enum": { "name": "Enum", @@ -177,8 +177,8 @@ "name": "ContentType", "kind": "class", "path": "omniread.core.content.ContentType", - "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nThis enum represents the declared or inferred media type of the content\nsource. It is primarily used for routing content to the appropriate\nparser or downstream consumer.", + "signature": "", + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", @@ -214,36 +214,36 @@ "name": "Content", "kind": "class", "path": "omniread.core.content.Content", - "signature": "", - "docstring": "Normalized representation of extracted content.\n\nA `Content` instance represents a raw content payload along with minimal\ncontextual metadata describing its origin and type.\n\nThis class is the **primary exchange format** between:\n- Scrapers\n- Parsers\n- Downstream consumers\n\nAttributes:\n raw: Raw content bytes as retrieved from the source.\n source: Identifier of the content origin (URL, file path, or logical name).\n content_type: Optional MIME type of the content, if known.\n metadata: Optional, implementation-defined metadata associated with\n the content (e.g., headers, encoding hints, extraction notes).", + "signature": "", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.core.content.Content.raw", "signature": null, - "docstring": null + "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.core.content.Content.source", "signature": null, - "docstring": null + "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.core.content.Content.content_type", "signature": null, - "docstring": null + "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.core.content.Content.metadata", "signature": null, - "docstring": null + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } } @@ -254,7 +254,7 @@ "kind": "module", "path": "omniread.core.parser", "signature": null, - "docstring": "Abstract parsing contracts for OmniRead.\n\nThis module defines the **format-agnostic parser interface** used to transform\nraw content into structured, typed representations.\n\nParsers are responsible for:\n- Interpreting a single `Content` instance\n- Validating compatibility with the content type\n- Producing a structured output suitable for downstream consumers\n\nParsers are not responsible for:\n- Fetching or acquiring content\n- Performing retries or error recovery\n- Managing multiple content sources", + "docstring": "# Summary\n\nAbstract parsing contracts for OmniRead.\n\nThis module defines the **format-agnostic parser interface** used to transform\nraw content into structured, typed representations.\n\nParsers are responsible for:\n\n- Interpreting a single `Content` instance\n- Validating compatibility with the content type\n- Producing a structured output suitable for downstream consumers\n\nParsers are not responsible for:\n\n- Fetching or acquiring content\n- Performing retries or error recovery\n- Managing multiple content sources", "members": { "ABC": { "name": "ABC", @@ -296,35 +296,35 @@ "kind": "class", "path": "omniread.core.parser.Content", "signature": "", - "docstring": "Normalized representation of extracted content.\n\nA `Content` instance represents a raw content payload along with minimal\ncontextual metadata describing its origin and type.\n\nThis class is the **primary exchange format** between:\n- Scrapers\n- Parsers\n- Downstream consumers\n\nAttributes:\n raw: Raw content bytes as retrieved from the source.\n source: Identifier of the content origin (URL, file path, or logical name).\n content_type: Optional MIME type of the content, if known.\n metadata: Optional, implementation-defined metadata associated with\n the content (e.g., headers, encoding hints, extraction notes).", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.core.parser.Content.raw", "signature": "", - "docstring": null + "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.core.parser.Content.source", "signature": "", - "docstring": null + "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.core.parser.Content.content_type", "signature": "", - "docstring": null + "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.core.parser.Content.metadata", "signature": "", - "docstring": null + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } }, @@ -333,7 +333,7 @@ "kind": "class", "path": "omniread.core.parser.ContentType", "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nThis enum represents the declared or inferred media type of the content\nsource. It is primarily used for routing content to the appropriate\nparser or downstream consumer.", + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", @@ -376,15 +376,15 @@ "name": "BaseParser", "kind": "class", "path": "omniread.core.parser.BaseParser", - "signature": "", - "docstring": "Base interface for all parsers.\n\nA parser is a self-contained object that owns the Content\nit is responsible for interpreting.\n\nImplementations must:\n- Declare supported content types via `supported_types`\n- Raise parsing-specific exceptions from `parse()`\n- Remain deterministic for a given input\n\nConsumers may rely on:\n- Early validation of content compatibility\n- Type-stable return values from `parse()`", + "signature": "", + "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", "members": { "supported_types": { "name": "supported_types", "kind": "attribute", "path": "omniread.core.parser.BaseParser.supported_types", "signature": null, - "docstring": "Set of content types supported by this parser.\n\nAn empty set indicates that the parser is content-type agnostic." + "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." }, "content": { "name": "content", @@ -397,15 +397,15 @@ "name": "parse", "kind": "function", "path": "omniread.core.parser.BaseParser.parse", - "signature": "", - "docstring": "Parse the owned content into structured output.\n\nImplementations must fully consume the provided content and\nreturn a deterministic, structured output.\n\nReturns:\n Parsed, structured representation.\n\nRaises:\n Exception: Parsing-specific errors as defined by the implementation." + "signature": "", + "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." }, "supports": { "name": "supports", "kind": "function", "path": "omniread.core.parser.BaseParser.supports", - "signature": "", - "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n True if the content type is supported; False otherwise." + "signature": "", + "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." } } } @@ -416,7 +416,7 @@ "kind": "module", "path": "omniread.core.scraper", "signature": null, - "docstring": "Abstract scraping contracts for OmniRead.\n\nThis module defines the **format-agnostic scraper interface** responsible for\nacquiring raw content from external sources.\n\nScrapers are responsible for:\n- Locating and retrieving raw content bytes\n- Attaching minimal contextual metadata\n- Returning normalized `Content` objects\n\nScrapers are explicitly NOT responsible for:\n- Parsing or interpreting content\n- Inferring structure or semantics\n- Performing content-type specific processing\n\nAll interpretation must be delegated to parsers.", + "docstring": "# Summary\n\nAbstract scraping contracts for OmniRead.\n\nThis module defines the **format-agnostic scraper interface** responsible for\nacquiring raw content from external sources.\n\nScrapers are responsible for:\n\n- Locating and retrieving raw content bytes\n- Attaching minimal contextual metadata\n- Returning normalized `Content` objects\n\nScrapers are explicitly NOT responsible for:\n\n- Parsing or interpreting content\n- Inferring structure or semantics\n- Performing content-type specific processing\n\nAll interpretation must be delegated to parsers.", "members": { "ABC": { "name": "ABC", @@ -458,35 +458,35 @@ "kind": "class", "path": "omniread.core.scraper.Content", "signature": "", - "docstring": "Normalized representation of extracted content.\n\nA `Content` instance represents a raw content payload along with minimal\ncontextual metadata describing its origin and type.\n\nThis class is the **primary exchange format** between:\n- Scrapers\n- Parsers\n- Downstream consumers\n\nAttributes:\n raw: Raw content bytes as retrieved from the source.\n source: Identifier of the content origin (URL, file path, or logical name).\n content_type: Optional MIME type of the content, if known.\n metadata: Optional, implementation-defined metadata associated with\n the content (e.g., headers, encoding hints, extraction notes).", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.core.scraper.Content.raw", "signature": "", - "docstring": null + "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.core.scraper.Content.source", "signature": "", - "docstring": null + "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.core.scraper.Content.content_type", "signature": "", - "docstring": null + "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.core.scraper.Content.metadata", "signature": "", - "docstring": null + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } }, @@ -494,15 +494,15 @@ "name": "BaseScraper", "kind": "class", "path": "omniread.core.scraper.BaseScraper", - "signature": "", - "docstring": "Base interface for all scrapers.\n\nA scraper is responsible ONLY for fetching raw content\n(bytes) from a source. It must not interpret or parse it.\n\nA scraper is a **stateless acquisition component** that retrieves raw\ncontent from a source and returns it as a `Content` object.\n\nScrapers define *how content is obtained*, not *what the content means*.\n\nImplementations may vary in:\n- Transport mechanism (HTTP, filesystem, cloud storage)\n- Authentication strategy\n- Retry and backoff behavior\n\nImplementations must not:\n- Parse content\n- Modify content semantics\n- Couple scraping logic to a specific parser", + "signature": "", + "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.core.scraper.BaseScraper.fetch", - "signature": "", - "docstring": "Fetch raw content from the given source.\n\nImplementations must retrieve the content referenced by `source`\nand return it as raw bytes wrapped in a `Content` object.\n\nArgs:\n source: Location identifier (URL, file path, S3 URI, etc.)\n metadata: Optional hints for the scraper (headers, auth, etc.)\n\nReturns:\n Content object containing raw bytes and metadata.\n - Raw content bytes\n - Source identifier\n - Optional metadata\n\nRaises:\n Exception: Retrieval-specific errors as defined by the implementation." + "signature": "", + "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." } } } diff --git a/mcp_docs/modules/omniread.core.parser.json b/mcp_docs/modules/omniread.core.parser.json index d30bb09..6df349d 100644 --- a/mcp_docs/modules/omniread.core.parser.json +++ b/mcp_docs/modules/omniread.core.parser.json @@ -2,7 +2,7 @@ "module": "omniread.core.parser", "content": { "path": "omniread.core.parser", - "docstring": "Abstract parsing contracts for OmniRead.\n\nThis module defines the **format-agnostic parser interface** used to transform\nraw content into structured, typed representations.\n\nParsers are responsible for:\n- Interpreting a single `Content` instance\n- Validating compatibility with the content type\n- Producing a structured output suitable for downstream consumers\n\nParsers are not responsible for:\n- Fetching or acquiring content\n- Performing retries or error recovery\n- Managing multiple content sources", + "docstring": "# Summary\n\nAbstract parsing contracts for OmniRead.\n\nThis module defines the **format-agnostic parser interface** used to transform\nraw content into structured, typed representations.\n\nParsers are responsible for:\n\n- Interpreting a single `Content` instance\n- Validating compatibility with the content type\n- Producing a structured output suitable for downstream consumers\n\nParsers are not responsible for:\n\n- Fetching or acquiring content\n- Performing retries or error recovery\n- Managing multiple content sources", "objects": { "ABC": { "name": "ABC", @@ -44,35 +44,35 @@ "kind": "class", "path": "omniread.core.parser.Content", "signature": "", - "docstring": "Normalized representation of extracted content.\n\nA `Content` instance represents a raw content payload along with minimal\ncontextual metadata describing its origin and type.\n\nThis class is the **primary exchange format** between:\n- Scrapers\n- Parsers\n- Downstream consumers\n\nAttributes:\n raw: Raw content bytes as retrieved from the source.\n source: Identifier of the content origin (URL, file path, or logical name).\n content_type: Optional MIME type of the content, if known.\n metadata: Optional, implementation-defined metadata associated with\n the content (e.g., headers, encoding hints, extraction notes).", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.core.parser.Content.raw", "signature": "", - "docstring": null + "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.core.parser.Content.source", "signature": "", - "docstring": null + "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.core.parser.Content.content_type", "signature": "", - "docstring": null + "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.core.parser.Content.metadata", "signature": "", - "docstring": null + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } }, @@ -81,7 +81,7 @@ "kind": "class", "path": "omniread.core.parser.ContentType", "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nThis enum represents the declared or inferred media type of the content\nsource. It is primarily used for routing content to the appropriate\nparser or downstream consumer.", + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", @@ -124,15 +124,15 @@ "name": "BaseParser", "kind": "class", "path": "omniread.core.parser.BaseParser", - "signature": "", - "docstring": "Base interface for all parsers.\n\nA parser is a self-contained object that owns the Content\nit is responsible for interpreting.\n\nImplementations must:\n- Declare supported content types via `supported_types`\n- Raise parsing-specific exceptions from `parse()`\n- Remain deterministic for a given input\n\nConsumers may rely on:\n- Early validation of content compatibility\n- Type-stable return values from `parse()`", + "signature": "", + "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", "members": { "supported_types": { "name": "supported_types", "kind": "attribute", "path": "omniread.core.parser.BaseParser.supported_types", "signature": null, - "docstring": "Set of content types supported by this parser.\n\nAn empty set indicates that the parser is content-type agnostic." + "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." }, "content": { "name": "content", @@ -145,15 +145,15 @@ "name": "parse", "kind": "function", "path": "omniread.core.parser.BaseParser.parse", - "signature": "", - "docstring": "Parse the owned content into structured output.\n\nImplementations must fully consume the provided content and\nreturn a deterministic, structured output.\n\nReturns:\n Parsed, structured representation.\n\nRaises:\n Exception: Parsing-specific errors as defined by the implementation." + "signature": "", + "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." }, "supports": { "name": "supports", "kind": "function", "path": "omniread.core.parser.BaseParser.supports", - "signature": "", - "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n True if the content type is supported; False otherwise." + "signature": "", + "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." } } } diff --git a/mcp_docs/modules/omniread.core.scraper.json b/mcp_docs/modules/omniread.core.scraper.json index 88ebbc9..2adcd11 100644 --- a/mcp_docs/modules/omniread.core.scraper.json +++ b/mcp_docs/modules/omniread.core.scraper.json @@ -2,7 +2,7 @@ "module": "omniread.core.scraper", "content": { "path": "omniread.core.scraper", - "docstring": "Abstract scraping contracts for OmniRead.\n\nThis module defines the **format-agnostic scraper interface** responsible for\nacquiring raw content from external sources.\n\nScrapers are responsible for:\n- Locating and retrieving raw content bytes\n- Attaching minimal contextual metadata\n- Returning normalized `Content` objects\n\nScrapers are explicitly NOT responsible for:\n- Parsing or interpreting content\n- Inferring structure or semantics\n- Performing content-type specific processing\n\nAll interpretation must be delegated to parsers.", + "docstring": "# Summary\n\nAbstract scraping contracts for OmniRead.\n\nThis module defines the **format-agnostic scraper interface** responsible for\nacquiring raw content from external sources.\n\nScrapers are responsible for:\n\n- Locating and retrieving raw content bytes\n- Attaching minimal contextual metadata\n- Returning normalized `Content` objects\n\nScrapers are explicitly NOT responsible for:\n\n- Parsing or interpreting content\n- Inferring structure or semantics\n- Performing content-type specific processing\n\nAll interpretation must be delegated to parsers.", "objects": { "ABC": { "name": "ABC", @@ -44,35 +44,35 @@ "kind": "class", "path": "omniread.core.scraper.Content", "signature": "", - "docstring": "Normalized representation of extracted content.\n\nA `Content` instance represents a raw content payload along with minimal\ncontextual metadata describing its origin and type.\n\nThis class is the **primary exchange format** between:\n- Scrapers\n- Parsers\n- Downstream consumers\n\nAttributes:\n raw: Raw content bytes as retrieved from the source.\n source: Identifier of the content origin (URL, file path, or logical name).\n content_type: Optional MIME type of the content, if known.\n metadata: Optional, implementation-defined metadata associated with\n the content (e.g., headers, encoding hints, extraction notes).", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.core.scraper.Content.raw", "signature": "", - "docstring": null + "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.core.scraper.Content.source", "signature": "", - "docstring": null + "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.core.scraper.Content.content_type", "signature": "", - "docstring": null + "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.core.scraper.Content.metadata", "signature": "", - "docstring": null + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } }, @@ -80,15 +80,15 @@ "name": "BaseScraper", "kind": "class", "path": "omniread.core.scraper.BaseScraper", - "signature": "", - "docstring": "Base interface for all scrapers.\n\nA scraper is responsible ONLY for fetching raw content\n(bytes) from a source. It must not interpret or parse it.\n\nA scraper is a **stateless acquisition component** that retrieves raw\ncontent from a source and returns it as a `Content` object.\n\nScrapers define *how content is obtained*, not *what the content means*.\n\nImplementations may vary in:\n- Transport mechanism (HTTP, filesystem, cloud storage)\n- Authentication strategy\n- Retry and backoff behavior\n\nImplementations must not:\n- Parse content\n- Modify content semantics\n- Couple scraping logic to a specific parser", + "signature": "", + "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.core.scraper.BaseScraper.fetch", - "signature": "", - "docstring": "Fetch raw content from the given source.\n\nImplementations must retrieve the content referenced by `source`\nand return it as raw bytes wrapped in a `Content` object.\n\nArgs:\n source: Location identifier (URL, file path, S3 URI, etc.)\n metadata: Optional hints for the scraper (headers, auth, etc.)\n\nReturns:\n Content object containing raw bytes and metadata.\n - Raw content bytes\n - Source identifier\n - Optional metadata\n\nRaises:\n Exception: Retrieval-specific errors as defined by the implementation." + "signature": "", + "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." } } } diff --git a/mcp_docs/modules/omniread.html.json b/mcp_docs/modules/omniread.html.json index 0c443d3..e079b2d 100644 --- a/mcp_docs/modules/omniread.html.json +++ b/mcp_docs/modules/omniread.html.json @@ -2,14 +2,14 @@ "module": "omniread.html", "content": { "path": "omniread.html", - "docstring": "HTML format implementation for OmniRead.\n\nThis package provides **HTML-specific implementations** of the core OmniRead\ncontracts defined in `omniread.core`.\n\nIt includes:\n- HTML parsers that interpret HTML content\n- HTML scrapers that retrieve HTML documents\n\nThis package:\n- Implements, but does not redefine, core contracts\n- May contain HTML-specific behavior and edge-case handling\n- Produces canonical content models defined in `omniread.core.content`\n\nConsumers should depend on `omniread.core` interfaces wherever possible and\nuse this package only when HTML-specific behavior is required.", + "docstring": "# Summary\n\nHTML format implementation for OmniRead.\n\nThis package provides **HTML-specific implementations** of the core OmniRead\ncontracts defined in `omniread.core`.\n\nIt includes:\n\n- HTML parsers that interpret HTML content.\n- HTML scrapers that retrieve HTML documents.\n\nKey characteristics:\n\n- Implements, but does not redefine, core contracts.\n- May contain HTML-specific behavior and edge-case handling.\n- Produces canonical content models defined in `omniread.core.content`.\n\nConsumers should depend on `omniread.core` interfaces wherever possible and\nuse this package only when HTML-specific behavior is required.\n\n---\n\n# Public API\n\n- `HTMLScraper`\n- `HTMLParser`\n\n---", "objects": { "HTMLScraper": { "name": "HTMLScraper", "kind": "class", "path": "omniread.html.HTMLScraper", "signature": "", - "docstring": "Base HTML scraper using httpx.\n\nThis scraper retrieves HTML documents over HTTP(S) and returns them\nas raw content wrapped in a `Content` object.\n\nFetches raw bytes and metadata only.\nThe scraper:\n- Uses `httpx.Client` for HTTP requests\n- Enforces an HTML content type\n- Preserves HTTP response metadata\n\nThe scraper does not:\n- Parse HTML\n- Perform retries or backoff\n- Handle non-HTML responses", + "docstring": "Base HTML scraper using `httpx`.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns\n them as raw content wrapped in a `Content` object.\n - Fetches raw bytes and metadata only.\n - The scraper uses `httpx.Client` for HTTP requests, enforces an\n HTML content type, and preserves HTTP response metadata.\n\n **Constraints:**\n\n - The scraper does not: Parse HTML, perform retries or backoff,\n handle non-HTML responses.", "members": { "content_type": { "name": "content_type", @@ -23,14 +23,14 @@ "kind": "function", "path": "omniread.html.HTMLScraper.validate_content_type", "signature": "", - "docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response: HTTP response returned by `httpx`.\n\nRaises:\n ValueError: If the `Content-Type` header is missing or does not\n indicate HTML content." + "docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content." }, "fetch": { "name": "fetch", "kind": "function", "path": "omniread.html.HTMLScraper.fetch", "signature": "", - "docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source: URL of the HTML document.\n metadata: Optional metadata to be merged into the returned content.\n\nReturns:\n A `Content` instance containing:\n - Raw HTML bytes\n - Source URL\n - HTML content type\n - HTTP response metadata\n\nRaises:\n httpx.HTTPError: If the HTTP request fails.\n ValueError: If the response is not valid HTML." + "docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML." } } }, @@ -39,7 +39,7 @@ "kind": "class", "path": "omniread.html.HTMLParser", "signature": "", - "docstring": "Base HTML parser.\n\nThis class extends the core `BaseParser` with HTML-specific behavior,\nincluding DOM parsing via BeautifulSoup and reusable extraction helpers.\n\nProvides reusable helpers for HTML extraction.\nConcrete parsers must explicitly define the return type.\n\nCharacteristics:\n- Accepts only HTML content\n- Owns a parsed BeautifulSoup DOM tree\n- Provides pure helper utilities for common HTML structures\n\nConcrete subclasses must:\n- Define the output type `T`\n- Implement the `parse()` method", + "docstring": "Base HTML parser.\n\nNotes:\n **Responsibilities:**\n\n - This class extends the core `BaseParser` with HTML-specific behavior,\n including DOM parsing via BeautifulSoup and reusable extraction helpers.\n - Provides reusable helpers for HTML extraction. Concrete parsers must\n explicitly define the return type.\n\n **Guarantees:**\n\n - Accepts only HTML content.\n - Owns a parsed BeautifulSoup DOM tree.\n - Provides pure helper utilities for common HTML structures.\n\n **Constraints:**\n\n - Concrete subclasses must define the output type `T` and implement\n the `parse()` method.", "members": { "supported_types": { "name": "supported_types", @@ -53,35 +53,35 @@ "kind": "function", "path": "omniread.html.HTMLParser.parse", "signature": "", - "docstring": "Fully parse the HTML content into structured output.\n\nImplementations must fully interpret the HTML DOM and return\na deterministic, structured output.\n\nReturns:\n Parsed representation of type `T`." + "docstring": "Fully parse the HTML content into structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the HTML DOM and return a\n deterministic, structured output." }, "parse_div": { "name": "parse_div", "kind": "function", "path": "omniread.html.HTMLParser.parse_div", "signature": "", - "docstring": "Extract normalized text from a `
` element.\n\nArgs:\n div: BeautifulSoup tag representing a `
`.\n separator: String used to separate text nodes.\n\nReturns:\n Flattened, whitespace-normalized text content." + "docstring": "Extract normalized text from a `
` element.\n\nArgs:\n div (Tag):\n BeautifulSoup tag representing a `
`.\n separator (str, optional):\n String used to separate text nodes.\n\nReturns:\n str:\n Flattened, whitespace-normalized text content." }, "parse_link": { "name": "parse_link", "kind": "function", "path": "omniread.html.HTMLParser.parse_link", "signature": "", - "docstring": "Extract the hyperlink reference from an `` element.\n\nArgs:\n a: BeautifulSoup tag representing an anchor.\n\nReturns:\n The value of the `href` attribute, or None if absent." + "docstring": "Extract the hyperlink reference from an `` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n Optional[str]:\n The value of the `href` attribute, or None if absent." }, "parse_table": { "name": "parse_table", "kind": "function", "path": "omniread.html.HTMLParser.parse_table", "signature": "", - "docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table: BeautifulSoup tag representing a ``.\n\nReturns:\n A list of rows, where each row is a list of cell text values." + "docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table (Tag):\n BeautifulSoup tag representing a `
`.\n\nReturns:\n list[list[str]]:\n A list of rows, where each row is a list of cell text values." }, "parse_meta": { "name": "parse_meta", "kind": "function", "path": "omniread.html.HTMLParser.parse_meta", "signature": "", - "docstring": "Extract high-level metadata from the HTML document.\n\nThis includes:\n- Document title\n- `` tag name/property → content mappings\n\nReturns:\n Dictionary containing extracted metadata." + "docstring": "Extract high-level metadata from the HTML document.\n\nReturns:\n dict[str, Any]:\n Dictionary containing extracted metadata.\n\nNotes:\n **Responsibilities:**\n\n - Extract high-level metadata from the HTML document.\n - This includes: Document title, `` tag name/property to\n content mappings." } } }, @@ -90,7 +90,7 @@ "kind": "module", "path": "omniread.html.parser", "signature": null, - "docstring": "HTML parser base implementations for OmniRead.\n\nThis module provides reusable HTML parsing utilities built on top of\nthe abstract parser contracts defined in `omniread.core.parser`.\n\nIt supplies:\n- Content-type enforcement for HTML inputs\n- BeautifulSoup initialization and lifecycle management\n- Common helper methods for extracting structured data from HTML elements\n\nConcrete parsers must subclass `HTMLParser` and implement the `parse()` method\nto return a structured representation appropriate for their use case.", + "docstring": "# Summary\n\nHTML parser base implementations for OmniRead.\n\nThis module provides reusable HTML parsing utilities built on top of\nthe abstract parser contracts defined in `omniread.core.parser`.\n\nIt supplies:\n\n- Content-type enforcement for HTML inputs\n- BeautifulSoup initialization and lifecycle management\n- Common helper methods for extracting structured data from HTML elements\n\nConcrete parsers must subclass `HTMLParser` and implement the `parse()` method\nto return a structured representation appropriate for their use case.", "members": { "Any": { "name": "Any", @@ -146,7 +146,7 @@ "kind": "class", "path": "omniread.html.parser.ContentType", "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nThis enum represents the declared or inferred media type of the content\nsource. It is primarily used for routing content to the appropriate\nparser or downstream consumer.", + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", @@ -183,35 +183,35 @@ "kind": "class", "path": "omniread.html.parser.Content", "signature": "", - "docstring": "Normalized representation of extracted content.\n\nA `Content` instance represents a raw content payload along with minimal\ncontextual metadata describing its origin and type.\n\nThis class is the **primary exchange format** between:\n- Scrapers\n- Parsers\n- Downstream consumers\n\nAttributes:\n raw: Raw content bytes as retrieved from the source.\n source: Identifier of the content origin (URL, file path, or logical name).\n content_type: Optional MIME type of the content, if known.\n metadata: Optional, implementation-defined metadata associated with\n the content (e.g., headers, encoding hints, extraction notes).", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.html.parser.Content.raw", "signature": "", - "docstring": null + "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.html.parser.Content.source", "signature": "", - "docstring": null + "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.html.parser.Content.content_type", "signature": "", - "docstring": null + "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.html.parser.Content.metadata", "signature": "", - "docstring": null + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } }, @@ -220,14 +220,14 @@ "kind": "class", "path": "omniread.html.parser.BaseParser", "signature": "", - "docstring": "Base interface for all parsers.\n\nA parser is a self-contained object that owns the Content\nit is responsible for interpreting.\n\nImplementations must:\n- Declare supported content types via `supported_types`\n- Raise parsing-specific exceptions from `parse()`\n- Remain deterministic for a given input\n\nConsumers may rely on:\n- Early validation of content compatibility\n- Type-stable return values from `parse()`", + "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", "members": { "supported_types": { "name": "supported_types", "kind": "attribute", "path": "omniread.html.parser.BaseParser.supported_types", "signature": "", - "docstring": "Set of content types supported by this parser.\n\nAn empty set indicates that the parser is content-type agnostic." + "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." }, "content": { "name": "content", @@ -241,14 +241,14 @@ "kind": "function", "path": "omniread.html.parser.BaseParser.parse", "signature": "", - "docstring": "Parse the owned content into structured output.\n\nImplementations must fully consume the provided content and\nreturn a deterministic, structured output.\n\nReturns:\n Parsed, structured representation.\n\nRaises:\n Exception: Parsing-specific errors as defined by the implementation." + "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." }, "supports": { "name": "supports", "kind": "function", "path": "omniread.html.parser.BaseParser.supports", "signature": "", - "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n True if the content type is supported; False otherwise." + "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." } } }, @@ -263,8 +263,8 @@ "name": "HTMLParser", "kind": "class", "path": "omniread.html.parser.HTMLParser", - "signature": "", - "docstring": "Base HTML parser.\n\nThis class extends the core `BaseParser` with HTML-specific behavior,\nincluding DOM parsing via BeautifulSoup and reusable extraction helpers.\n\nProvides reusable helpers for HTML extraction.\nConcrete parsers must explicitly define the return type.\n\nCharacteristics:\n- Accepts only HTML content\n- Owns a parsed BeautifulSoup DOM tree\n- Provides pure helper utilities for common HTML structures\n\nConcrete subclasses must:\n- Define the output type `T`\n- Implement the `parse()` method", + "signature": "", + "docstring": "Base HTML parser.\n\nNotes:\n **Responsibilities:**\n\n - This class extends the core `BaseParser` with HTML-specific behavior,\n including DOM parsing via BeautifulSoup and reusable extraction helpers.\n - Provides reusable helpers for HTML extraction. Concrete parsers must\n explicitly define the return type.\n\n **Guarantees:**\n\n - Accepts only HTML content.\n - Owns a parsed BeautifulSoup DOM tree.\n - Provides pure helper utilities for common HTML structures.\n\n **Constraints:**\n\n - Concrete subclasses must define the output type `T` and implement\n the `parse()` method.", "members": { "supported_types": { "name": "supported_types", @@ -277,36 +277,36 @@ "name": "parse", "kind": "function", "path": "omniread.html.parser.HTMLParser.parse", - "signature": "", - "docstring": "Fully parse the HTML content into structured output.\n\nImplementations must fully interpret the HTML DOM and return\na deterministic, structured output.\n\nReturns:\n Parsed representation of type `T`." + "signature": "", + "docstring": "Fully parse the HTML content into structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the HTML DOM and return a\n deterministic, structured output." }, "parse_div": { "name": "parse_div", "kind": "function", "path": "omniread.html.parser.HTMLParser.parse_div", - "signature": "", - "docstring": "Extract normalized text from a `
` element.\n\nArgs:\n div: BeautifulSoup tag representing a `
`.\n separator: String used to separate text nodes.\n\nReturns:\n Flattened, whitespace-normalized text content." + "signature": "", + "docstring": "Extract normalized text from a `
` element.\n\nArgs:\n div (Tag):\n BeautifulSoup tag representing a `
`.\n separator (str, optional):\n String used to separate text nodes.\n\nReturns:\n str:\n Flattened, whitespace-normalized text content." }, "parse_link": { "name": "parse_link", "kind": "function", "path": "omniread.html.parser.HTMLParser.parse_link", - "signature": "", - "docstring": "Extract the hyperlink reference from an `` element.\n\nArgs:\n a: BeautifulSoup tag representing an anchor.\n\nReturns:\n The value of the `href` attribute, or None if absent." + "signature": "", + "docstring": "Extract the hyperlink reference from an `` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n Optional[str]:\n The value of the `href` attribute, or None if absent." }, "parse_table": { "name": "parse_table", "kind": "function", "path": "omniread.html.parser.HTMLParser.parse_table", - "signature": "", - "docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table: BeautifulSoup tag representing a `
`.\n\nReturns:\n A list of rows, where each row is a list of cell text values." + "signature": "", + "docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table (Tag):\n BeautifulSoup tag representing a `
`.\n\nReturns:\n list[list[str]]:\n A list of rows, where each row is a list of cell text values." }, "parse_meta": { "name": "parse_meta", "kind": "function", "path": "omniread.html.parser.HTMLParser.parse_meta", - "signature": "", - "docstring": "Extract high-level metadata from the HTML document.\n\nThis includes:\n- Document title\n- `` tag name/property → content mappings\n\nReturns:\n Dictionary containing extracted metadata." + "signature": "", + "docstring": "Extract high-level metadata from the HTML document.\n\nReturns:\n dict[str, Any]:\n Dictionary containing extracted metadata.\n\nNotes:\n **Responsibilities:**\n\n - Extract high-level metadata from the HTML document.\n - This includes: Document title, `` tag name/property to\n content mappings." } } }, @@ -331,7 +331,7 @@ "kind": "module", "path": "omniread.html.scraper", "signature": null, - "docstring": "HTML scraping implementation for OmniRead.\n\nThis module provides an HTTP-based scraper for retrieving HTML documents.\nIt implements the core `BaseScraper` contract using `httpx` as the transport\nlayer.\n\nThis scraper is responsible for:\n- Fetching raw HTML bytes over HTTP(S)\n- Validating response content type\n- Attaching HTTP metadata to the returned content\n\nThis scraper is not responsible for:\n- Parsing or interpreting HTML\n- Retrying failed requests\n- Managing crawl policies or rate limiting", + "docstring": "# Summary\n\nHTML scraping implementation for OmniRead.\n\nThis module provides an HTTP-based scraper for retrieving HTML documents.\nIt implements the core `BaseScraper` contract using `httpx` as the transport\nlayer.\n\nThis scraper is responsible for:\n\n- Fetching raw HTML bytes over HTTP(S)\n- Validating response content type\n- Attaching HTTP metadata to the returned content\n\nThis scraper is not responsible for:\n\n- Parsing or interpreting HTML\n- Retrying failed requests\n- Managing crawl policies or rate limiting", "members": { "httpx": { "name": "httpx", @@ -366,35 +366,35 @@ "kind": "class", "path": "omniread.html.scraper.Content", "signature": "", - "docstring": "Normalized representation of extracted content.\n\nA `Content` instance represents a raw content payload along with minimal\ncontextual metadata describing its origin and type.\n\nThis class is the **primary exchange format** between:\n- Scrapers\n- Parsers\n- Downstream consumers\n\nAttributes:\n raw: Raw content bytes as retrieved from the source.\n source: Identifier of the content origin (URL, file path, or logical name).\n content_type: Optional MIME type of the content, if known.\n metadata: Optional, implementation-defined metadata associated with\n the content (e.g., headers, encoding hints, extraction notes).", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.html.scraper.Content.raw", "signature": "", - "docstring": null + "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.html.scraper.Content.source", "signature": "", - "docstring": null + "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.html.scraper.Content.content_type", "signature": "", - "docstring": null + "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.html.scraper.Content.metadata", "signature": "", - "docstring": null + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } }, @@ -403,7 +403,7 @@ "kind": "class", "path": "omniread.html.scraper.ContentType", "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nThis enum represents the declared or inferred media type of the content\nsource. It is primarily used for routing content to the appropriate\nparser or downstream consumer.", + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", @@ -440,14 +440,14 @@ "kind": "class", "path": "omniread.html.scraper.BaseScraper", "signature": "", - "docstring": "Base interface for all scrapers.\n\nA scraper is responsible ONLY for fetching raw content\n(bytes) from a source. It must not interpret or parse it.\n\nA scraper is a **stateless acquisition component** that retrieves raw\ncontent from a source and returns it as a `Content` object.\n\nScrapers define *how content is obtained*, not *what the content means*.\n\nImplementations may vary in:\n- Transport mechanism (HTTP, filesystem, cloud storage)\n- Authentication strategy\n- Retry and backoff behavior\n\nImplementations must not:\n- Parse content\n- Modify content semantics\n- Couple scraping logic to a specific parser", + "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.html.scraper.BaseScraper.fetch", "signature": "", - "docstring": "Fetch raw content from the given source.\n\nImplementations must retrieve the content referenced by `source`\nand return it as raw bytes wrapped in a `Content` object.\n\nArgs:\n source: Location identifier (URL, file path, S3 URI, etc.)\n metadata: Optional hints for the scraper (headers, auth, etc.)\n\nReturns:\n Content object containing raw bytes and metadata.\n - Raw content bytes\n - Source identifier\n - Optional metadata\n\nRaises:\n Exception: Retrieval-specific errors as defined by the implementation." + "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." } } }, @@ -455,8 +455,8 @@ "name": "HTMLScraper", "kind": "class", "path": "omniread.html.scraper.HTMLScraper", - "signature": "", - "docstring": "Base HTML scraper using httpx.\n\nThis scraper retrieves HTML documents over HTTP(S) and returns them\nas raw content wrapped in a `Content` object.\n\nFetches raw bytes and metadata only.\nThe scraper:\n- Uses `httpx.Client` for HTTP requests\n- Enforces an HTML content type\n- Preserves HTTP response metadata\n\nThe scraper does not:\n- Parse HTML\n- Perform retries or backoff\n- Handle non-HTML responses", + "signature": "", + "docstring": "Base HTML scraper using `httpx`.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns\n them as raw content wrapped in a `Content` object.\n - Fetches raw bytes and metadata only.\n - The scraper uses `httpx.Client` for HTTP requests, enforces an\n HTML content type, and preserves HTTP response metadata.\n\n **Constraints:**\n\n - The scraper does not: Parse HTML, perform retries or backoff,\n handle non-HTML responses.", "members": { "content_type": { "name": "content_type", @@ -469,15 +469,15 @@ "name": "validate_content_type", "kind": "function", "path": "omniread.html.scraper.HTMLScraper.validate_content_type", - "signature": "", - "docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response: HTTP response returned by `httpx`.\n\nRaises:\n ValueError: If the `Content-Type` header is missing or does not\n indicate HTML content." + "signature": "", + "docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content." }, "fetch": { "name": "fetch", "kind": "function", "path": "omniread.html.scraper.HTMLScraper.fetch", - "signature": "", - "docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source: URL of the HTML document.\n metadata: Optional metadata to be merged into the returned content.\n\nReturns:\n A `Content` instance containing:\n - Raw HTML bytes\n - Source URL\n - HTML content type\n - HTTP response metadata\n\nRaises:\n httpx.HTTPError: If the HTTP request fails.\n ValueError: If the response is not valid HTML." + "signature": "", + "docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML." } } } diff --git a/mcp_docs/modules/omniread.html.parser.json b/mcp_docs/modules/omniread.html.parser.json index 147bfdc..c607d2d 100644 --- a/mcp_docs/modules/omniread.html.parser.json +++ b/mcp_docs/modules/omniread.html.parser.json @@ -2,7 +2,7 @@ "module": "omniread.html.parser", "content": { "path": "omniread.html.parser", - "docstring": "HTML parser base implementations for OmniRead.\n\nThis module provides reusable HTML parsing utilities built on top of\nthe abstract parser contracts defined in `omniread.core.parser`.\n\nIt supplies:\n- Content-type enforcement for HTML inputs\n- BeautifulSoup initialization and lifecycle management\n- Common helper methods for extracting structured data from HTML elements\n\nConcrete parsers must subclass `HTMLParser` and implement the `parse()` method\nto return a structured representation appropriate for their use case.", + "docstring": "# Summary\n\nHTML parser base implementations for OmniRead.\n\nThis module provides reusable HTML parsing utilities built on top of\nthe abstract parser contracts defined in `omniread.core.parser`.\n\nIt supplies:\n\n- Content-type enforcement for HTML inputs\n- BeautifulSoup initialization and lifecycle management\n- Common helper methods for extracting structured data from HTML elements\n\nConcrete parsers must subclass `HTMLParser` and implement the `parse()` method\nto return a structured representation appropriate for their use case.", "objects": { "Any": { "name": "Any", @@ -58,7 +58,7 @@ "kind": "class", "path": "omniread.html.parser.ContentType", "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nThis enum represents the declared or inferred media type of the content\nsource. It is primarily used for routing content to the appropriate\nparser or downstream consumer.", + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", @@ -95,35 +95,35 @@ "kind": "class", "path": "omniread.html.parser.Content", "signature": "", - "docstring": "Normalized representation of extracted content.\n\nA `Content` instance represents a raw content payload along with minimal\ncontextual metadata describing its origin and type.\n\nThis class is the **primary exchange format** between:\n- Scrapers\n- Parsers\n- Downstream consumers\n\nAttributes:\n raw: Raw content bytes as retrieved from the source.\n source: Identifier of the content origin (URL, file path, or logical name).\n content_type: Optional MIME type of the content, if known.\n metadata: Optional, implementation-defined metadata associated with\n the content (e.g., headers, encoding hints, extraction notes).", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.html.parser.Content.raw", "signature": "", - "docstring": null + "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.html.parser.Content.source", "signature": "", - "docstring": null + "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.html.parser.Content.content_type", "signature": "", - "docstring": null + "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.html.parser.Content.metadata", "signature": "", - "docstring": null + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } }, @@ -132,14 +132,14 @@ "kind": "class", "path": "omniread.html.parser.BaseParser", "signature": "", - "docstring": "Base interface for all parsers.\n\nA parser is a self-contained object that owns the Content\nit is responsible for interpreting.\n\nImplementations must:\n- Declare supported content types via `supported_types`\n- Raise parsing-specific exceptions from `parse()`\n- Remain deterministic for a given input\n\nConsumers may rely on:\n- Early validation of content compatibility\n- Type-stable return values from `parse()`", + "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", "members": { "supported_types": { "name": "supported_types", "kind": "attribute", "path": "omniread.html.parser.BaseParser.supported_types", "signature": "", - "docstring": "Set of content types supported by this parser.\n\nAn empty set indicates that the parser is content-type agnostic." + "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." }, "content": { "name": "content", @@ -153,14 +153,14 @@ "kind": "function", "path": "omniread.html.parser.BaseParser.parse", "signature": "", - "docstring": "Parse the owned content into structured output.\n\nImplementations must fully consume the provided content and\nreturn a deterministic, structured output.\n\nReturns:\n Parsed, structured representation.\n\nRaises:\n Exception: Parsing-specific errors as defined by the implementation." + "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." }, "supports": { "name": "supports", "kind": "function", "path": "omniread.html.parser.BaseParser.supports", "signature": "", - "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n True if the content type is supported; False otherwise." + "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." } } }, @@ -175,8 +175,8 @@ "name": "HTMLParser", "kind": "class", "path": "omniread.html.parser.HTMLParser", - "signature": "", - "docstring": "Base HTML parser.\n\nThis class extends the core `BaseParser` with HTML-specific behavior,\nincluding DOM parsing via BeautifulSoup and reusable extraction helpers.\n\nProvides reusable helpers for HTML extraction.\nConcrete parsers must explicitly define the return type.\n\nCharacteristics:\n- Accepts only HTML content\n- Owns a parsed BeautifulSoup DOM tree\n- Provides pure helper utilities for common HTML structures\n\nConcrete subclasses must:\n- Define the output type `T`\n- Implement the `parse()` method", + "signature": "", + "docstring": "Base HTML parser.\n\nNotes:\n **Responsibilities:**\n\n - This class extends the core `BaseParser` with HTML-specific behavior,\n including DOM parsing via BeautifulSoup and reusable extraction helpers.\n - Provides reusable helpers for HTML extraction. Concrete parsers must\n explicitly define the return type.\n\n **Guarantees:**\n\n - Accepts only HTML content.\n - Owns a parsed BeautifulSoup DOM tree.\n - Provides pure helper utilities for common HTML structures.\n\n **Constraints:**\n\n - Concrete subclasses must define the output type `T` and implement\n the `parse()` method.", "members": { "supported_types": { "name": "supported_types", @@ -189,36 +189,36 @@ "name": "parse", "kind": "function", "path": "omniread.html.parser.HTMLParser.parse", - "signature": "", - "docstring": "Fully parse the HTML content into structured output.\n\nImplementations must fully interpret the HTML DOM and return\na deterministic, structured output.\n\nReturns:\n Parsed representation of type `T`." + "signature": "", + "docstring": "Fully parse the HTML content into structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the HTML DOM and return a\n deterministic, structured output." }, "parse_div": { "name": "parse_div", "kind": "function", "path": "omniread.html.parser.HTMLParser.parse_div", - "signature": "", - "docstring": "Extract normalized text from a `
` element.\n\nArgs:\n div: BeautifulSoup tag representing a `
`.\n separator: String used to separate text nodes.\n\nReturns:\n Flattened, whitespace-normalized text content." + "signature": "", + "docstring": "Extract normalized text from a `
` element.\n\nArgs:\n div (Tag):\n BeautifulSoup tag representing a `
`.\n separator (str, optional):\n String used to separate text nodes.\n\nReturns:\n str:\n Flattened, whitespace-normalized text content." }, "parse_link": { "name": "parse_link", "kind": "function", "path": "omniread.html.parser.HTMLParser.parse_link", - "signature": "", - "docstring": "Extract the hyperlink reference from an `` element.\n\nArgs:\n a: BeautifulSoup tag representing an anchor.\n\nReturns:\n The value of the `href` attribute, or None if absent." + "signature": "", + "docstring": "Extract the hyperlink reference from an `` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n Optional[str]:\n The value of the `href` attribute, or None if absent." }, "parse_table": { "name": "parse_table", "kind": "function", "path": "omniread.html.parser.HTMLParser.parse_table", - "signature": "", - "docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table: BeautifulSoup tag representing a `
`.\n\nReturns:\n A list of rows, where each row is a list of cell text values." + "signature": "", + "docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table (Tag):\n BeautifulSoup tag representing a `
`.\n\nReturns:\n list[list[str]]:\n A list of rows, where each row is a list of cell text values." }, "parse_meta": { "name": "parse_meta", "kind": "function", "path": "omniread.html.parser.HTMLParser.parse_meta", - "signature": "", - "docstring": "Extract high-level metadata from the HTML document.\n\nThis includes:\n- Document title\n- `` tag name/property → content mappings\n\nReturns:\n Dictionary containing extracted metadata." + "signature": "", + "docstring": "Extract high-level metadata from the HTML document.\n\nReturns:\n dict[str, Any]:\n Dictionary containing extracted metadata.\n\nNotes:\n **Responsibilities:**\n\n - Extract high-level metadata from the HTML document.\n - This includes: Document title, `` tag name/property to\n content mappings." } } }, diff --git a/mcp_docs/modules/omniread.html.scraper.json b/mcp_docs/modules/omniread.html.scraper.json index b360df9..e1865d9 100644 --- a/mcp_docs/modules/omniread.html.scraper.json +++ b/mcp_docs/modules/omniread.html.scraper.json @@ -2,7 +2,7 @@ "module": "omniread.html.scraper", "content": { "path": "omniread.html.scraper", - "docstring": "HTML scraping implementation for OmniRead.\n\nThis module provides an HTTP-based scraper for retrieving HTML documents.\nIt implements the core `BaseScraper` contract using `httpx` as the transport\nlayer.\n\nThis scraper is responsible for:\n- Fetching raw HTML bytes over HTTP(S)\n- Validating response content type\n- Attaching HTTP metadata to the returned content\n\nThis scraper is not responsible for:\n- Parsing or interpreting HTML\n- Retrying failed requests\n- Managing crawl policies or rate limiting", + "docstring": "# Summary\n\nHTML scraping implementation for OmniRead.\n\nThis module provides an HTTP-based scraper for retrieving HTML documents.\nIt implements the core `BaseScraper` contract using `httpx` as the transport\nlayer.\n\nThis scraper is responsible for:\n\n- Fetching raw HTML bytes over HTTP(S)\n- Validating response content type\n- Attaching HTTP metadata to the returned content\n\nThis scraper is not responsible for:\n\n- Parsing or interpreting HTML\n- Retrying failed requests\n- Managing crawl policies or rate limiting", "objects": { "httpx": { "name": "httpx", @@ -37,35 +37,35 @@ "kind": "class", "path": "omniread.html.scraper.Content", "signature": "", - "docstring": "Normalized representation of extracted content.\n\nA `Content` instance represents a raw content payload along with minimal\ncontextual metadata describing its origin and type.\n\nThis class is the **primary exchange format** between:\n- Scrapers\n- Parsers\n- Downstream consumers\n\nAttributes:\n raw: Raw content bytes as retrieved from the source.\n source: Identifier of the content origin (URL, file path, or logical name).\n content_type: Optional MIME type of the content, if known.\n metadata: Optional, implementation-defined metadata associated with\n the content (e.g., headers, encoding hints, extraction notes).", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.html.scraper.Content.raw", "signature": "", - "docstring": null + "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.html.scraper.Content.source", "signature": "", - "docstring": null + "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.html.scraper.Content.content_type", "signature": "", - "docstring": null + "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.html.scraper.Content.metadata", "signature": "", - "docstring": null + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } }, @@ -74,7 +74,7 @@ "kind": "class", "path": "omniread.html.scraper.ContentType", "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nThis enum represents the declared or inferred media type of the content\nsource. It is primarily used for routing content to the appropriate\nparser or downstream consumer.", + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", @@ -111,14 +111,14 @@ "kind": "class", "path": "omniread.html.scraper.BaseScraper", "signature": "", - "docstring": "Base interface for all scrapers.\n\nA scraper is responsible ONLY for fetching raw content\n(bytes) from a source. It must not interpret or parse it.\n\nA scraper is a **stateless acquisition component** that retrieves raw\ncontent from a source and returns it as a `Content` object.\n\nScrapers define *how content is obtained*, not *what the content means*.\n\nImplementations may vary in:\n- Transport mechanism (HTTP, filesystem, cloud storage)\n- Authentication strategy\n- Retry and backoff behavior\n\nImplementations must not:\n- Parse content\n- Modify content semantics\n- Couple scraping logic to a specific parser", + "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.html.scraper.BaseScraper.fetch", "signature": "", - "docstring": "Fetch raw content from the given source.\n\nImplementations must retrieve the content referenced by `source`\nand return it as raw bytes wrapped in a `Content` object.\n\nArgs:\n source: Location identifier (URL, file path, S3 URI, etc.)\n metadata: Optional hints for the scraper (headers, auth, etc.)\n\nReturns:\n Content object containing raw bytes and metadata.\n - Raw content bytes\n - Source identifier\n - Optional metadata\n\nRaises:\n Exception: Retrieval-specific errors as defined by the implementation." + "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." } } }, @@ -126,8 +126,8 @@ "name": "HTMLScraper", "kind": "class", "path": "omniread.html.scraper.HTMLScraper", - "signature": "", - "docstring": "Base HTML scraper using httpx.\n\nThis scraper retrieves HTML documents over HTTP(S) and returns them\nas raw content wrapped in a `Content` object.\n\nFetches raw bytes and metadata only.\nThe scraper:\n- Uses `httpx.Client` for HTTP requests\n- Enforces an HTML content type\n- Preserves HTTP response metadata\n\nThe scraper does not:\n- Parse HTML\n- Perform retries or backoff\n- Handle non-HTML responses", + "signature": "", + "docstring": "Base HTML scraper using `httpx`.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns\n them as raw content wrapped in a `Content` object.\n - Fetches raw bytes and metadata only.\n - The scraper uses `httpx.Client` for HTTP requests, enforces an\n HTML content type, and preserves HTTP response metadata.\n\n **Constraints:**\n\n - The scraper does not: Parse HTML, perform retries or backoff,\n handle non-HTML responses.", "members": { "content_type": { "name": "content_type", @@ -140,15 +140,15 @@ "name": "validate_content_type", "kind": "function", "path": "omniread.html.scraper.HTMLScraper.validate_content_type", - "signature": "", - "docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response: HTTP response returned by `httpx`.\n\nRaises:\n ValueError: If the `Content-Type` header is missing or does not\n indicate HTML content." + "signature": "", + "docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content." }, "fetch": { "name": "fetch", "kind": "function", "path": "omniread.html.scraper.HTMLScraper.fetch", - "signature": "", - "docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source: URL of the HTML document.\n metadata: Optional metadata to be merged into the returned content.\n\nReturns:\n A `Content` instance containing:\n - Raw HTML bytes\n - Source URL\n - HTML content type\n - HTTP response metadata\n\nRaises:\n httpx.HTTPError: If the HTTP request fails.\n ValueError: If the response is not valid HTML." + "signature": "", + "docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML." } } } diff --git a/mcp_docs/modules/omniread.json b/mcp_docs/modules/omniread.json index a321fcf..8b122a6 100644 --- a/mcp_docs/modules/omniread.json +++ b/mcp_docs/modules/omniread.json @@ -2,42 +2,42 @@ "module": "omniread", "content": { "path": "omniread", - "docstring": "OmniRead — format-agnostic content acquisition and parsing framework.\n\nOmniRead provides a **cleanly layered architecture** for fetching, parsing,\nand normalizing content from heterogeneous sources such as HTML documents\nand PDF files.\n\nThe library is structured around three core concepts:\n\n1. **Content**\n A canonical, format-agnostic container representing raw content bytes\n and minimal contextual metadata.\n\n2. **Scrapers**\n Components responsible for *acquiring* raw content from a source\n (HTTP, filesystem, object storage, etc.). Scrapers never interpret\n content.\n\n3. **Parsers**\n Components responsible for *interpreting* acquired content and\n converting it into structured, typed representations.\n\nOmniRead deliberately separates these responsibilities to ensure:\n- Clear boundaries between IO and interpretation\n- Replaceable implementations per format\n- Predictable, testable behavior\n\n----------------------------------------------------------------------\nInstallation\n----------------------------------------------------------------------\n\nInstall OmniRead using pip:\n\n pip install omniread\n\nOr with Poetry:\n\n poetry add omniread\n\n----------------------------------------------------------------------\nBasic Usage\n----------------------------------------------------------------------\n\nHTML example:\n\n from omniread import HTMLScraper, HTMLParser\n\n scraper = HTMLScraper()\n content = scraper.fetch(\"https://example.com\")\n\n class TitleParser(HTMLParser[str]):\n def parse(self) -> str:\n return self._soup.title.string\n\n parser = TitleParser(content)\n title = parser.parse()\n\nPDF example:\n\n from omniread import FileSystemPDFClient, PDFScraper, PDFParser\n from pathlib import Path\n\n client = FileSystemPDFClient()\n scraper = PDFScraper(client=client)\n content = scraper.fetch(Path(\"document.pdf\"))\n\n class TextPDFParser(PDFParser[str]):\n def parse(self) -> str:\n # implement PDF text extraction\n ...\n\n parser = TextPDFParser(content)\n result = parser.parse()\n\n----------------------------------------------------------------------\nPublic API Surface\n----------------------------------------------------------------------\n\nThis module re-exports the **recommended public entry points** of OmniRead.\n\nConsumers are encouraged to import from this namespace rather than from\nformat-specific submodules directly, unless advanced customization is\nrequired.\n\nCore:\n- Content\n- ContentType\n\nHTML:\n- HTMLScraper\n- HTMLParser\n\nPDF:\n- FileSystemPDFClient\n- PDFScraper\n- PDFParser\n\n## Core Philosophy\n\n`OmniRead` is designed as a **decoupled content engine**:\n\n1. **Separation of Concerns**: Scrapers *fetch*, Parsers *interpret*. Neither knows about the other.\n2. **Normalized Exchange**: All components communicate via the `Content` model, ensuring a consistent contract.\n3. **Format Agnosticism**: The core logic is independent of whether the input is HTML, PDF, or JSON.\n\n## Documentation Design\n\nFor those extending `OmniRead`, follow these \"AI-Native\" docstring principles:\n\n### For Humans\n- **Clear Contracts**: Explicitly state what a component is and is NOT responsible for.\n- **Runnable Examples**: Include small, logical snippets in the package `__init__.py`.\n\n### For LLMs\n- **Structured Models**: Use dataclasses and enums for core data to ensure clean MCP JSON representation.\n- **Type Safety**: All public APIs must be fully typed and have corresponding `.pyi` stubs.\n- **Detailed Raises**: Include `: description` pairs in the `Raises` section to help agents handle errors gracefully.", + "docstring": "# Summary\n\n`OmniRead` — format-agnostic content acquisition and parsing framework.\n\n`OmniRead` provides a **cleanly layered architecture** for fetching, parsing,\nand normalizing content from heterogeneous sources such as HTML documents\nand PDF files.\n\nThe library is structured around three core concepts:\n\n1. **`Content`**: A canonical, format-agnostic container representing raw content\n bytes and minimal contextual metadata.\n2. **`Scrapers`**: Components responsible for *acquiring* raw content from a\n source (HTTP, filesystem, object storage, etc.). `Scrapers` never interpret\n content.\n3. **`Parsers`**: Components responsible for *interpreting* acquired content and\n converting it into structured, typed representations.\n\n`OmniRead` deliberately separates these responsibilities to ensure:\n\n- Clear boundaries between IO and interpretation.\n- Replaceable implementations per format.\n- Predictable, testable behavior.\n\n# Installation\n\nInstall `OmniRead` using pip:\n\n```bash\npip install omniread\n```\n\nInstall OmniRead using Poetry:\n```bash\npoetry add omniread\n```\n\n---\n\n## Quick start\n\nExample:\n HTML example:\n ```python\n from omniread import HTMLScraper, HTMLParser\n\n scraper = HTMLScraper()\n content = scraper.fetch(\"https://example.com\")\n\n class TitleParser(HTMLParser[str]):\n def parse(self) -> str:\n return self._soup.title.string\n\n parser = TitleParser(content)\n title = parser.parse()\n ```\n\n PDF example:\n ```python\n from omniread import FileSystemPDFClient, PDFScraper, PDFParser\n from pathlib import Path\n\n client = FileSystemPDFClient()\n scraper = PDFScraper(client=client)\n content = scraper.fetch(Path(\"document.pdf\"))\n\n class TextPDFParser(PDFParser[str]):\n def parse(self) -> str:\n # implement PDF text extraction\n ...\n\n parser = TextPDFParser(content)\n result = parser.parse()\n ```\n\n---\n\n# Public API\n\nThis module re-exports the **recommended public entry points** of OmniRead.\nConsumers are encouraged to import from this namespace rather than from\nformat-specific submodules directly, unless advanced customization is\nrequired.\n\n- `Content`: Canonical content model.\n- `ContentType`: Supported media types.\n- `HTMLScraper`: HTTP-based HTML acquisition.\n- `HTMLParser`: Base parser for HTML DOM interpretation.\n- `FileSystemPDFClient`: Local filesystem PDF access.\n- `PDFScraper`: PDF-specific content acquisition.\n- `PDFParser`: Base parser for PDF binary interpretation.\n\n---\n\n# Core Philosophy\n\n`OmniRead` is designed as a **decoupled content engine**:\n\n1. **Separation of Concerns**: Scrapers *fetch*, Parsers *interpret*. Neither\n knows about the other.\n2. **Normalized Exchange**: All components communicate via the `Content` model,\n ensuring a consistent contract.\n3. **Format Agnosticism**: The core logic is independent of whether the input\n is HTML, PDF, or JSON.\n\n---", "objects": { "Content": { "name": "Content", "kind": "class", "path": "omniread.Content", "signature": "", - "docstring": "Normalized representation of extracted content.\n\nA `Content` instance represents a raw content payload along with minimal\ncontextual metadata describing its origin and type.\n\nThis class is the **primary exchange format** between:\n- Scrapers\n- Parsers\n- Downstream consumers\n\nAttributes:\n raw: Raw content bytes as retrieved from the source.\n source: Identifier of the content origin (URL, file path, or logical name).\n content_type: Optional MIME type of the content, if known.\n metadata: Optional, implementation-defined metadata associated with\n the content (e.g., headers, encoding hints, extraction notes).", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.Content.raw", "signature": "", - "docstring": null + "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.Content.source", "signature": "", - "docstring": null + "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.Content.content_type", "signature": "", - "docstring": null + "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.Content.metadata", "signature": "", - "docstring": null + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } }, @@ -46,7 +46,7 @@ "kind": "class", "path": "omniread.ContentType", "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nThis enum represents the declared or inferred media type of the content\nsource. It is primarily used for routing content to the appropriate\nparser or downstream consumer.", + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", @@ -83,7 +83,7 @@ "kind": "class", "path": "omniread.HTMLScraper", "signature": "", - "docstring": "Base HTML scraper using httpx.\n\nThis scraper retrieves HTML documents over HTTP(S) and returns them\nas raw content wrapped in a `Content` object.\n\nFetches raw bytes and metadata only.\nThe scraper:\n- Uses `httpx.Client` for HTTP requests\n- Enforces an HTML content type\n- Preserves HTTP response metadata\n\nThe scraper does not:\n- Parse HTML\n- Perform retries or backoff\n- Handle non-HTML responses", + "docstring": "Base HTML scraper using `httpx`.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns\n them as raw content wrapped in a `Content` object.\n - Fetches raw bytes and metadata only.\n - The scraper uses `httpx.Client` for HTTP requests, enforces an\n HTML content type, and preserves HTTP response metadata.\n\n **Constraints:**\n\n - The scraper does not: Parse HTML, perform retries or backoff,\n handle non-HTML responses.", "members": { "content_type": { "name": "content_type", @@ -97,14 +97,14 @@ "kind": "function", "path": "omniread.HTMLScraper.validate_content_type", "signature": "", - "docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response: HTTP response returned by `httpx`.\n\nRaises:\n ValueError: If the `Content-Type` header is missing or does not\n indicate HTML content." + "docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content." }, "fetch": { "name": "fetch", "kind": "function", "path": "omniread.HTMLScraper.fetch", "signature": "", - "docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source: URL of the HTML document.\n metadata: Optional metadata to be merged into the returned content.\n\nReturns:\n A `Content` instance containing:\n - Raw HTML bytes\n - Source URL\n - HTML content type\n - HTTP response metadata\n\nRaises:\n httpx.HTTPError: If the HTTP request fails.\n ValueError: If the response is not valid HTML." + "docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML." } } }, @@ -113,7 +113,7 @@ "kind": "class", "path": "omniread.HTMLParser", "signature": "", - "docstring": "Base HTML parser.\n\nThis class extends the core `BaseParser` with HTML-specific behavior,\nincluding DOM parsing via BeautifulSoup and reusable extraction helpers.\n\nProvides reusable helpers for HTML extraction.\nConcrete parsers must explicitly define the return type.\n\nCharacteristics:\n- Accepts only HTML content\n- Owns a parsed BeautifulSoup DOM tree\n- Provides pure helper utilities for common HTML structures\n\nConcrete subclasses must:\n- Define the output type `T`\n- Implement the `parse()` method", + "docstring": "Base HTML parser.\n\nNotes:\n **Responsibilities:**\n\n - This class extends the core `BaseParser` with HTML-specific behavior,\n including DOM parsing via BeautifulSoup and reusable extraction helpers.\n - Provides reusable helpers for HTML extraction. Concrete parsers must\n explicitly define the return type.\n\n **Guarantees:**\n\n - Accepts only HTML content.\n - Owns a parsed BeautifulSoup DOM tree.\n - Provides pure helper utilities for common HTML structures.\n\n **Constraints:**\n\n - Concrete subclasses must define the output type `T` and implement\n the `parse()` method.", "members": { "supported_types": { "name": "supported_types", @@ -127,35 +127,35 @@ "kind": "function", "path": "omniread.HTMLParser.parse", "signature": "", - "docstring": "Fully parse the HTML content into structured output.\n\nImplementations must fully interpret the HTML DOM and return\na deterministic, structured output.\n\nReturns:\n Parsed representation of type `T`." + "docstring": "Fully parse the HTML content into structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the HTML DOM and return a\n deterministic, structured output." }, "parse_div": { "name": "parse_div", "kind": "function", "path": "omniread.HTMLParser.parse_div", "signature": "", - "docstring": "Extract normalized text from a `
` element.\n\nArgs:\n div: BeautifulSoup tag representing a `
`.\n separator: String used to separate text nodes.\n\nReturns:\n Flattened, whitespace-normalized text content." + "docstring": "Extract normalized text from a `
` element.\n\nArgs:\n div (Tag):\n BeautifulSoup tag representing a `
`.\n separator (str, optional):\n String used to separate text nodes.\n\nReturns:\n str:\n Flattened, whitespace-normalized text content." }, "parse_link": { "name": "parse_link", "kind": "function", "path": "omniread.HTMLParser.parse_link", "signature": "", - "docstring": "Extract the hyperlink reference from an `` element.\n\nArgs:\n a: BeautifulSoup tag representing an anchor.\n\nReturns:\n The value of the `href` attribute, or None if absent." + "docstring": "Extract the hyperlink reference from an `` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n Optional[str]:\n The value of the `href` attribute, or None if absent." }, "parse_table": { "name": "parse_table", "kind": "function", "path": "omniread.HTMLParser.parse_table", "signature": "", - "docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table: BeautifulSoup tag representing a `
`.\n\nReturns:\n A list of rows, where each row is a list of cell text values." + "docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table (Tag):\n BeautifulSoup tag representing a `
`.\n\nReturns:\n list[list[str]]:\n A list of rows, where each row is a list of cell text values." }, "parse_meta": { "name": "parse_meta", "kind": "function", "path": "omniread.HTMLParser.parse_meta", "signature": "", - "docstring": "Extract high-level metadata from the HTML document.\n\nThis includes:\n- Document title\n- `` tag name/property → content mappings\n\nReturns:\n Dictionary containing extracted metadata." + "docstring": "Extract high-level metadata from the HTML document.\n\nReturns:\n dict[str, Any]:\n Dictionary containing extracted metadata.\n\nNotes:\n **Responsibilities:**\n\n - Extract high-level metadata from the HTML document.\n - This includes: Document title, `` tag name/property to\n content mappings." } } }, @@ -164,14 +164,14 @@ "kind": "class", "path": "omniread.FileSystemPDFClient", "signature": "", - "docstring": "PDF client that reads from the local filesystem.\n\nThis client reads PDF files directly from the disk and returns their raw\nbinary contents.", + "docstring": "PDF client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads PDF files directly from the disk and returns\n their raw binary contents.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.FileSystemPDFClient.fetch", "signature": "", - "docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path: Filesystem path to the PDF file.\n\nReturns:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError: If the path does not exist.\n ValueError: If the path exists but is not a file." + "docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the PDF file.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." } } }, @@ -180,14 +180,14 @@ "kind": "class", "path": "omniread.PDFScraper", "signature": "", - "docstring": "Scraper for PDF sources.\n\nDelegates byte retrieval to a PDF client and normalizes\noutput into Content.\n\nThe scraper:\n- Does not perform parsing or interpretation\n- Does not assume a specific storage backend\n- Preserves caller-provided metadata", + "docstring": "Scraper for PDF sources.\n\nNotes:\n **Responsibilities:**\n\n - Delegates byte retrieval to a PDF client and normalizes output\n into `Content`.\n - Preserves caller-provided metadata.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.PDFScraper.fetch", "signature": "", - "docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source: Identifier of the PDF source as understood by the\n configured PDF client.\n metadata: Optional metadata to attach to the returned content.\n\nReturns:\n A `Content` instance containing:\n - Raw PDF bytes\n - Source identifier\n - PDF content type\n - Optional metadata\n\nRaises:\n Exception: Retrieval-specific errors raised by the PDF client." + "docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client." } } }, @@ -196,7 +196,7 @@ "kind": "class", "path": "omniread.PDFParser", "signature": "", - "docstring": "Base PDF parser.\n\nThis class enforces PDF content-type compatibility and provides the\nextension point for implementing concrete PDF parsing strategies.\n\nConcrete implementations must define:\n- Define the output type `T`\n- Implement the `parse()` method", + "docstring": "Base PDF parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces PDF content-type compatibility and provides\n the extension point for implementing concrete PDF parsing strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", "members": { "supported_types": { "name": "supported_types", @@ -210,7 +210,7 @@ "kind": "function", "path": "omniread.PDFParser.parse", "signature": "", - "docstring": "Parse PDF content into a structured output.\n\nImplementations must fully interpret the PDF binary payload and\nreturn a deterministic, structured output.\n\nReturns:\n Parsed representation of type `T`.\n\nRaises:\n Exception: Parsing-specific errors as defined by the implementation." + "docstring": "Parse PDF content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the PDF binary payload and\n return a deterministic, structured output." } } }, @@ -219,42 +219,42 @@ "kind": "module", "path": "omniread.core", "signature": null, - "docstring": "Core domain contracts for OmniRead.\n\nThis package defines the **format-agnostic domain layer** of OmniRead.\nIt exposes canonical content models and abstract interfaces that are\nimplemented by format-specific modules (HTML, PDF, etc.).\n\nPublic exports from this package are considered **stable contracts** and\nare safe for downstream consumers to depend on.\n\nSubmodules:\n- content: Canonical content models and enums\n- parser: Abstract parsing contracts\n- scraper: Abstract scraping contracts\n\nFormat-specific behavior must not be introduced at this layer.", + "docstring": "# Summary\n\nCore domain contracts for OmniRead.\n\nThis package defines the **format-agnostic domain layer** of OmniRead.\nIt exposes canonical content models and abstract interfaces that are\nimplemented by format-specific modules (HTML, PDF, etc.).\n\nPublic exports from this package are considered **stable contracts** and\nare safe for downstream consumers to depend on.\n\nSubmodules:\n\n- `content`: Canonical content models and enums.\n- `parser`: Abstract parsing contracts.\n- `scraper`: Abstract scraping contracts.\n\nFormat-specific behavior must not be introduced at this layer.\n\n---\n\n# Public API\n\n- `Content`\n- `ContentType`\n\n---", "members": { "Content": { "name": "Content", "kind": "class", "path": "omniread.core.Content", "signature": "", - "docstring": "Normalized representation of extracted content.\n\nA `Content` instance represents a raw content payload along with minimal\ncontextual metadata describing its origin and type.\n\nThis class is the **primary exchange format** between:\n- Scrapers\n- Parsers\n- Downstream consumers\n\nAttributes:\n raw: Raw content bytes as retrieved from the source.\n source: Identifier of the content origin (URL, file path, or logical name).\n content_type: Optional MIME type of the content, if known.\n metadata: Optional, implementation-defined metadata associated with\n the content (e.g., headers, encoding hints, extraction notes).", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.core.Content.raw", "signature": "", - "docstring": null + "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.core.Content.source", "signature": "", - "docstring": null + "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.core.Content.content_type", "signature": "", - "docstring": null + "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.core.Content.metadata", "signature": "", - "docstring": null + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } }, @@ -263,7 +263,7 @@ "kind": "class", "path": "omniread.core.ContentType", "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nThis enum represents the declared or inferred media type of the content\nsource. It is primarily used for routing content to the appropriate\nparser or downstream consumer.", + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", @@ -300,14 +300,14 @@ "kind": "class", "path": "omniread.core.BaseParser", "signature": "", - "docstring": "Base interface for all parsers.\n\nA parser is a self-contained object that owns the Content\nit is responsible for interpreting.\n\nImplementations must:\n- Declare supported content types via `supported_types`\n- Raise parsing-specific exceptions from `parse()`\n- Remain deterministic for a given input\n\nConsumers may rely on:\n- Early validation of content compatibility\n- Type-stable return values from `parse()`", + "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", "members": { "supported_types": { "name": "supported_types", "kind": "attribute", "path": "omniread.core.BaseParser.supported_types", "signature": "", - "docstring": "Set of content types supported by this parser.\n\nAn empty set indicates that the parser is content-type agnostic." + "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." }, "content": { "name": "content", @@ -321,14 +321,14 @@ "kind": "function", "path": "omniread.core.BaseParser.parse", "signature": "", - "docstring": "Parse the owned content into structured output.\n\nImplementations must fully consume the provided content and\nreturn a deterministic, structured output.\n\nReturns:\n Parsed, structured representation.\n\nRaises:\n Exception: Parsing-specific errors as defined by the implementation." + "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." }, "supports": { "name": "supports", "kind": "function", "path": "omniread.core.BaseParser.supports", "signature": "", - "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n True if the content type is supported; False otherwise." + "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." } } }, @@ -337,14 +337,14 @@ "kind": "class", "path": "omniread.core.BaseScraper", "signature": "", - "docstring": "Base interface for all scrapers.\n\nA scraper is responsible ONLY for fetching raw content\n(bytes) from a source. It must not interpret or parse it.\n\nA scraper is a **stateless acquisition component** that retrieves raw\ncontent from a source and returns it as a `Content` object.\n\nScrapers define *how content is obtained*, not *what the content means*.\n\nImplementations may vary in:\n- Transport mechanism (HTTP, filesystem, cloud storage)\n- Authentication strategy\n- Retry and backoff behavior\n\nImplementations must not:\n- Parse content\n- Modify content semantics\n- Couple scraping logic to a specific parser", + "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.core.BaseScraper.fetch", "signature": "", - "docstring": "Fetch raw content from the given source.\n\nImplementations must retrieve the content referenced by `source`\nand return it as raw bytes wrapped in a `Content` object.\n\nArgs:\n source: Location identifier (URL, file path, S3 URI, etc.)\n metadata: Optional hints for the scraper (headers, auth, etc.)\n\nReturns:\n Content object containing raw bytes and metadata.\n - Raw content bytes\n - Source identifier\n - Optional metadata\n\nRaises:\n Exception: Retrieval-specific errors as defined by the implementation." + "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." } } }, @@ -353,7 +353,7 @@ "kind": "module", "path": "omniread.core.content", "signature": null, - "docstring": "Canonical content models for OmniRead.\n\nThis module defines the **format-agnostic content representation** used across\nall parsers and scrapers in OmniRead.\n\nThe models defined here represent *what* was extracted, not *how* it was\nretrieved or parsed. Format-specific behavior and metadata must not alter\nthe semantic meaning of these models.", + "docstring": "# Summary\n\nCanonical content models for OmniRead.\n\nThis module defines the **format-agnostic content representation** used across\nall parsers and scrapers in OmniRead.\n\nThe models defined here represent *what* was extracted, not *how* it was\nretrieved or parsed. Format-specific behavior and metadata must not alter\nthe semantic meaning of these models.", "members": { "Enum": { "name": "Enum", @@ -394,8 +394,8 @@ "name": "ContentType", "kind": "class", "path": "omniread.core.content.ContentType", - "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nThis enum represents the declared or inferred media type of the content\nsource. It is primarily used for routing content to the appropriate\nparser or downstream consumer.", + "signature": "", + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", @@ -431,36 +431,36 @@ "name": "Content", "kind": "class", "path": "omniread.core.content.Content", - "signature": "", - "docstring": "Normalized representation of extracted content.\n\nA `Content` instance represents a raw content payload along with minimal\ncontextual metadata describing its origin and type.\n\nThis class is the **primary exchange format** between:\n- Scrapers\n- Parsers\n- Downstream consumers\n\nAttributes:\n raw: Raw content bytes as retrieved from the source.\n source: Identifier of the content origin (URL, file path, or logical name).\n content_type: Optional MIME type of the content, if known.\n metadata: Optional, implementation-defined metadata associated with\n the content (e.g., headers, encoding hints, extraction notes).", + "signature": "", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.core.content.Content.raw", "signature": null, - "docstring": null + "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.core.content.Content.source", "signature": null, - "docstring": null + "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.core.content.Content.content_type", "signature": null, - "docstring": null + "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.core.content.Content.metadata", "signature": null, - "docstring": null + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } } @@ -471,7 +471,7 @@ "kind": "module", "path": "omniread.core.parser", "signature": null, - "docstring": "Abstract parsing contracts for OmniRead.\n\nThis module defines the **format-agnostic parser interface** used to transform\nraw content into structured, typed representations.\n\nParsers are responsible for:\n- Interpreting a single `Content` instance\n- Validating compatibility with the content type\n- Producing a structured output suitable for downstream consumers\n\nParsers are not responsible for:\n- Fetching or acquiring content\n- Performing retries or error recovery\n- Managing multiple content sources", + "docstring": "# Summary\n\nAbstract parsing contracts for OmniRead.\n\nThis module defines the **format-agnostic parser interface** used to transform\nraw content into structured, typed representations.\n\nParsers are responsible for:\n\n- Interpreting a single `Content` instance\n- Validating compatibility with the content type\n- Producing a structured output suitable for downstream consumers\n\nParsers are not responsible for:\n\n- Fetching or acquiring content\n- Performing retries or error recovery\n- Managing multiple content sources", "members": { "ABC": { "name": "ABC", @@ -513,35 +513,35 @@ "kind": "class", "path": "omniread.core.parser.Content", "signature": "", - "docstring": "Normalized representation of extracted content.\n\nA `Content` instance represents a raw content payload along with minimal\ncontextual metadata describing its origin and type.\n\nThis class is the **primary exchange format** between:\n- Scrapers\n- Parsers\n- Downstream consumers\n\nAttributes:\n raw: Raw content bytes as retrieved from the source.\n source: Identifier of the content origin (URL, file path, or logical name).\n content_type: Optional MIME type of the content, if known.\n metadata: Optional, implementation-defined metadata associated with\n the content (e.g., headers, encoding hints, extraction notes).", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.core.parser.Content.raw", "signature": "", - "docstring": null + "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.core.parser.Content.source", "signature": "", - "docstring": null + "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.core.parser.Content.content_type", "signature": "", - "docstring": null + "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.core.parser.Content.metadata", "signature": "", - "docstring": null + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } }, @@ -550,7 +550,7 @@ "kind": "class", "path": "omniread.core.parser.ContentType", "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nThis enum represents the declared or inferred media type of the content\nsource. It is primarily used for routing content to the appropriate\nparser or downstream consumer.", + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", @@ -593,15 +593,15 @@ "name": "BaseParser", "kind": "class", "path": "omniread.core.parser.BaseParser", - "signature": "", - "docstring": "Base interface for all parsers.\n\nA parser is a self-contained object that owns the Content\nit is responsible for interpreting.\n\nImplementations must:\n- Declare supported content types via `supported_types`\n- Raise parsing-specific exceptions from `parse()`\n- Remain deterministic for a given input\n\nConsumers may rely on:\n- Early validation of content compatibility\n- Type-stable return values from `parse()`", + "signature": "", + "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", "members": { "supported_types": { "name": "supported_types", "kind": "attribute", "path": "omniread.core.parser.BaseParser.supported_types", "signature": null, - "docstring": "Set of content types supported by this parser.\n\nAn empty set indicates that the parser is content-type agnostic." + "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." }, "content": { "name": "content", @@ -614,15 +614,15 @@ "name": "parse", "kind": "function", "path": "omniread.core.parser.BaseParser.parse", - "signature": "", - "docstring": "Parse the owned content into structured output.\n\nImplementations must fully consume the provided content and\nreturn a deterministic, structured output.\n\nReturns:\n Parsed, structured representation.\n\nRaises:\n Exception: Parsing-specific errors as defined by the implementation." + "signature": "", + "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." }, "supports": { "name": "supports", "kind": "function", "path": "omniread.core.parser.BaseParser.supports", - "signature": "", - "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n True if the content type is supported; False otherwise." + "signature": "", + "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." } } } @@ -633,7 +633,7 @@ "kind": "module", "path": "omniread.core.scraper", "signature": null, - "docstring": "Abstract scraping contracts for OmniRead.\n\nThis module defines the **format-agnostic scraper interface** responsible for\nacquiring raw content from external sources.\n\nScrapers are responsible for:\n- Locating and retrieving raw content bytes\n- Attaching minimal contextual metadata\n- Returning normalized `Content` objects\n\nScrapers are explicitly NOT responsible for:\n- Parsing or interpreting content\n- Inferring structure or semantics\n- Performing content-type specific processing\n\nAll interpretation must be delegated to parsers.", + "docstring": "# Summary\n\nAbstract scraping contracts for OmniRead.\n\nThis module defines the **format-agnostic scraper interface** responsible for\nacquiring raw content from external sources.\n\nScrapers are responsible for:\n\n- Locating and retrieving raw content bytes\n- Attaching minimal contextual metadata\n- Returning normalized `Content` objects\n\nScrapers are explicitly NOT responsible for:\n\n- Parsing or interpreting content\n- Inferring structure or semantics\n- Performing content-type specific processing\n\nAll interpretation must be delegated to parsers.", "members": { "ABC": { "name": "ABC", @@ -675,35 +675,35 @@ "kind": "class", "path": "omniread.core.scraper.Content", "signature": "", - "docstring": "Normalized representation of extracted content.\n\nA `Content` instance represents a raw content payload along with minimal\ncontextual metadata describing its origin and type.\n\nThis class is the **primary exchange format** between:\n- Scrapers\n- Parsers\n- Downstream consumers\n\nAttributes:\n raw: Raw content bytes as retrieved from the source.\n source: Identifier of the content origin (URL, file path, or logical name).\n content_type: Optional MIME type of the content, if known.\n metadata: Optional, implementation-defined metadata associated with\n the content (e.g., headers, encoding hints, extraction notes).", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.core.scraper.Content.raw", "signature": "", - "docstring": null + "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.core.scraper.Content.source", "signature": "", - "docstring": null + "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.core.scraper.Content.content_type", "signature": "", - "docstring": null + "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.core.scraper.Content.metadata", "signature": "", - "docstring": null + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } }, @@ -711,15 +711,15 @@ "name": "BaseScraper", "kind": "class", "path": "omniread.core.scraper.BaseScraper", - "signature": "", - "docstring": "Base interface for all scrapers.\n\nA scraper is responsible ONLY for fetching raw content\n(bytes) from a source. It must not interpret or parse it.\n\nA scraper is a **stateless acquisition component** that retrieves raw\ncontent from a source and returns it as a `Content` object.\n\nScrapers define *how content is obtained*, not *what the content means*.\n\nImplementations may vary in:\n- Transport mechanism (HTTP, filesystem, cloud storage)\n- Authentication strategy\n- Retry and backoff behavior\n\nImplementations must not:\n- Parse content\n- Modify content semantics\n- Couple scraping logic to a specific parser", + "signature": "", + "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.core.scraper.BaseScraper.fetch", - "signature": "", - "docstring": "Fetch raw content from the given source.\n\nImplementations must retrieve the content referenced by `source`\nand return it as raw bytes wrapped in a `Content` object.\n\nArgs:\n source: Location identifier (URL, file path, S3 URI, etc.)\n metadata: Optional hints for the scraper (headers, auth, etc.)\n\nReturns:\n Content object containing raw bytes and metadata.\n - Raw content bytes\n - Source identifier\n - Optional metadata\n\nRaises:\n Exception: Retrieval-specific errors as defined by the implementation." + "signature": "", + "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." } } } @@ -732,14 +732,14 @@ "kind": "module", "path": "omniread.html", "signature": null, - "docstring": "HTML format implementation for OmniRead.\n\nThis package provides **HTML-specific implementations** of the core OmniRead\ncontracts defined in `omniread.core`.\n\nIt includes:\n- HTML parsers that interpret HTML content\n- HTML scrapers that retrieve HTML documents\n\nThis package:\n- Implements, but does not redefine, core contracts\n- May contain HTML-specific behavior and edge-case handling\n- Produces canonical content models defined in `omniread.core.content`\n\nConsumers should depend on `omniread.core` interfaces wherever possible and\nuse this package only when HTML-specific behavior is required.", + "docstring": "# Summary\n\nHTML format implementation for OmniRead.\n\nThis package provides **HTML-specific implementations** of the core OmniRead\ncontracts defined in `omniread.core`.\n\nIt includes:\n\n- HTML parsers that interpret HTML content.\n- HTML scrapers that retrieve HTML documents.\n\nKey characteristics:\n\n- Implements, but does not redefine, core contracts.\n- May contain HTML-specific behavior and edge-case handling.\n- Produces canonical content models defined in `omniread.core.content`.\n\nConsumers should depend on `omniread.core` interfaces wherever possible and\nuse this package only when HTML-specific behavior is required.\n\n---\n\n# Public API\n\n- `HTMLScraper`\n- `HTMLParser`\n\n---", "members": { "HTMLScraper": { "name": "HTMLScraper", "kind": "class", "path": "omniread.html.HTMLScraper", "signature": "", - "docstring": "Base HTML scraper using httpx.\n\nThis scraper retrieves HTML documents over HTTP(S) and returns them\nas raw content wrapped in a `Content` object.\n\nFetches raw bytes and metadata only.\nThe scraper:\n- Uses `httpx.Client` for HTTP requests\n- Enforces an HTML content type\n- Preserves HTTP response metadata\n\nThe scraper does not:\n- Parse HTML\n- Perform retries or backoff\n- Handle non-HTML responses", + "docstring": "Base HTML scraper using `httpx`.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns\n them as raw content wrapped in a `Content` object.\n - Fetches raw bytes and metadata only.\n - The scraper uses `httpx.Client` for HTTP requests, enforces an\n HTML content type, and preserves HTTP response metadata.\n\n **Constraints:**\n\n - The scraper does not: Parse HTML, perform retries or backoff,\n handle non-HTML responses.", "members": { "content_type": { "name": "content_type", @@ -753,14 +753,14 @@ "kind": "function", "path": "omniread.html.HTMLScraper.validate_content_type", "signature": "", - "docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response: HTTP response returned by `httpx`.\n\nRaises:\n ValueError: If the `Content-Type` header is missing or does not\n indicate HTML content." + "docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content." }, "fetch": { "name": "fetch", "kind": "function", "path": "omniread.html.HTMLScraper.fetch", "signature": "", - "docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source: URL of the HTML document.\n metadata: Optional metadata to be merged into the returned content.\n\nReturns:\n A `Content` instance containing:\n - Raw HTML bytes\n - Source URL\n - HTML content type\n - HTTP response metadata\n\nRaises:\n httpx.HTTPError: If the HTTP request fails.\n ValueError: If the response is not valid HTML." + "docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML." } } }, @@ -769,7 +769,7 @@ "kind": "class", "path": "omniread.html.HTMLParser", "signature": "", - "docstring": "Base HTML parser.\n\nThis class extends the core `BaseParser` with HTML-specific behavior,\nincluding DOM parsing via BeautifulSoup and reusable extraction helpers.\n\nProvides reusable helpers for HTML extraction.\nConcrete parsers must explicitly define the return type.\n\nCharacteristics:\n- Accepts only HTML content\n- Owns a parsed BeautifulSoup DOM tree\n- Provides pure helper utilities for common HTML structures\n\nConcrete subclasses must:\n- Define the output type `T`\n- Implement the `parse()` method", + "docstring": "Base HTML parser.\n\nNotes:\n **Responsibilities:**\n\n - This class extends the core `BaseParser` with HTML-specific behavior,\n including DOM parsing via BeautifulSoup and reusable extraction helpers.\n - Provides reusable helpers for HTML extraction. Concrete parsers must\n explicitly define the return type.\n\n **Guarantees:**\n\n - Accepts only HTML content.\n - Owns a parsed BeautifulSoup DOM tree.\n - Provides pure helper utilities for common HTML structures.\n\n **Constraints:**\n\n - Concrete subclasses must define the output type `T` and implement\n the `parse()` method.", "members": { "supported_types": { "name": "supported_types", @@ -783,35 +783,35 @@ "kind": "function", "path": "omniread.html.HTMLParser.parse", "signature": "", - "docstring": "Fully parse the HTML content into structured output.\n\nImplementations must fully interpret the HTML DOM and return\na deterministic, structured output.\n\nReturns:\n Parsed representation of type `T`." + "docstring": "Fully parse the HTML content into structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the HTML DOM and return a\n deterministic, structured output." }, "parse_div": { "name": "parse_div", "kind": "function", "path": "omniread.html.HTMLParser.parse_div", "signature": "", - "docstring": "Extract normalized text from a `
` element.\n\nArgs:\n div: BeautifulSoup tag representing a `
`.\n separator: String used to separate text nodes.\n\nReturns:\n Flattened, whitespace-normalized text content." + "docstring": "Extract normalized text from a `
` element.\n\nArgs:\n div (Tag):\n BeautifulSoup tag representing a `
`.\n separator (str, optional):\n String used to separate text nodes.\n\nReturns:\n str:\n Flattened, whitespace-normalized text content." }, "parse_link": { "name": "parse_link", "kind": "function", "path": "omniread.html.HTMLParser.parse_link", "signature": "", - "docstring": "Extract the hyperlink reference from an `` element.\n\nArgs:\n a: BeautifulSoup tag representing an anchor.\n\nReturns:\n The value of the `href` attribute, or None if absent." + "docstring": "Extract the hyperlink reference from an `` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n Optional[str]:\n The value of the `href` attribute, or None if absent." }, "parse_table": { "name": "parse_table", "kind": "function", "path": "omniread.html.HTMLParser.parse_table", "signature": "", - "docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table: BeautifulSoup tag representing a `
`.\n\nReturns:\n A list of rows, where each row is a list of cell text values." + "docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table (Tag):\n BeautifulSoup tag representing a `
`.\n\nReturns:\n list[list[str]]:\n A list of rows, where each row is a list of cell text values." }, "parse_meta": { "name": "parse_meta", "kind": "function", "path": "omniread.html.HTMLParser.parse_meta", "signature": "", - "docstring": "Extract high-level metadata from the HTML document.\n\nThis includes:\n- Document title\n- `` tag name/property → content mappings\n\nReturns:\n Dictionary containing extracted metadata." + "docstring": "Extract high-level metadata from the HTML document.\n\nReturns:\n dict[str, Any]:\n Dictionary containing extracted metadata.\n\nNotes:\n **Responsibilities:**\n\n - Extract high-level metadata from the HTML document.\n - This includes: Document title, `` tag name/property to\n content mappings." } } }, @@ -820,7 +820,7 @@ "kind": "module", "path": "omniread.html.parser", "signature": null, - "docstring": "HTML parser base implementations for OmniRead.\n\nThis module provides reusable HTML parsing utilities built on top of\nthe abstract parser contracts defined in `omniread.core.parser`.\n\nIt supplies:\n- Content-type enforcement for HTML inputs\n- BeautifulSoup initialization and lifecycle management\n- Common helper methods for extracting structured data from HTML elements\n\nConcrete parsers must subclass `HTMLParser` and implement the `parse()` method\nto return a structured representation appropriate for their use case.", + "docstring": "# Summary\n\nHTML parser base implementations for OmniRead.\n\nThis module provides reusable HTML parsing utilities built on top of\nthe abstract parser contracts defined in `omniread.core.parser`.\n\nIt supplies:\n\n- Content-type enforcement for HTML inputs\n- BeautifulSoup initialization and lifecycle management\n- Common helper methods for extracting structured data from HTML elements\n\nConcrete parsers must subclass `HTMLParser` and implement the `parse()` method\nto return a structured representation appropriate for their use case.", "members": { "Any": { "name": "Any", @@ -876,7 +876,7 @@ "kind": "class", "path": "omniread.html.parser.ContentType", "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nThis enum represents the declared or inferred media type of the content\nsource. It is primarily used for routing content to the appropriate\nparser or downstream consumer.", + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", @@ -913,35 +913,35 @@ "kind": "class", "path": "omniread.html.parser.Content", "signature": "", - "docstring": "Normalized representation of extracted content.\n\nA `Content` instance represents a raw content payload along with minimal\ncontextual metadata describing its origin and type.\n\nThis class is the **primary exchange format** between:\n- Scrapers\n- Parsers\n- Downstream consumers\n\nAttributes:\n raw: Raw content bytes as retrieved from the source.\n source: Identifier of the content origin (URL, file path, or logical name).\n content_type: Optional MIME type of the content, if known.\n metadata: Optional, implementation-defined metadata associated with\n the content (e.g., headers, encoding hints, extraction notes).", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.html.parser.Content.raw", "signature": "", - "docstring": null + "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.html.parser.Content.source", "signature": "", - "docstring": null + "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.html.parser.Content.content_type", "signature": "", - "docstring": null + "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.html.parser.Content.metadata", "signature": "", - "docstring": null + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } }, @@ -950,14 +950,14 @@ "kind": "class", "path": "omniread.html.parser.BaseParser", "signature": "", - "docstring": "Base interface for all parsers.\n\nA parser is a self-contained object that owns the Content\nit is responsible for interpreting.\n\nImplementations must:\n- Declare supported content types via `supported_types`\n- Raise parsing-specific exceptions from `parse()`\n- Remain deterministic for a given input\n\nConsumers may rely on:\n- Early validation of content compatibility\n- Type-stable return values from `parse()`", + "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", "members": { "supported_types": { "name": "supported_types", "kind": "attribute", "path": "omniread.html.parser.BaseParser.supported_types", "signature": "", - "docstring": "Set of content types supported by this parser.\n\nAn empty set indicates that the parser is content-type agnostic." + "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." }, "content": { "name": "content", @@ -971,14 +971,14 @@ "kind": "function", "path": "omniread.html.parser.BaseParser.parse", "signature": "", - "docstring": "Parse the owned content into structured output.\n\nImplementations must fully consume the provided content and\nreturn a deterministic, structured output.\n\nReturns:\n Parsed, structured representation.\n\nRaises:\n Exception: Parsing-specific errors as defined by the implementation." + "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." }, "supports": { "name": "supports", "kind": "function", "path": "omniread.html.parser.BaseParser.supports", "signature": "", - "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n True if the content type is supported; False otherwise." + "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." } } }, @@ -993,8 +993,8 @@ "name": "HTMLParser", "kind": "class", "path": "omniread.html.parser.HTMLParser", - "signature": "", - "docstring": "Base HTML parser.\n\nThis class extends the core `BaseParser` with HTML-specific behavior,\nincluding DOM parsing via BeautifulSoup and reusable extraction helpers.\n\nProvides reusable helpers for HTML extraction.\nConcrete parsers must explicitly define the return type.\n\nCharacteristics:\n- Accepts only HTML content\n- Owns a parsed BeautifulSoup DOM tree\n- Provides pure helper utilities for common HTML structures\n\nConcrete subclasses must:\n- Define the output type `T`\n- Implement the `parse()` method", + "signature": "", + "docstring": "Base HTML parser.\n\nNotes:\n **Responsibilities:**\n\n - This class extends the core `BaseParser` with HTML-specific behavior,\n including DOM parsing via BeautifulSoup and reusable extraction helpers.\n - Provides reusable helpers for HTML extraction. Concrete parsers must\n explicitly define the return type.\n\n **Guarantees:**\n\n - Accepts only HTML content.\n - Owns a parsed BeautifulSoup DOM tree.\n - Provides pure helper utilities for common HTML structures.\n\n **Constraints:**\n\n - Concrete subclasses must define the output type `T` and implement\n the `parse()` method.", "members": { "supported_types": { "name": "supported_types", @@ -1007,36 +1007,36 @@ "name": "parse", "kind": "function", "path": "omniread.html.parser.HTMLParser.parse", - "signature": "", - "docstring": "Fully parse the HTML content into structured output.\n\nImplementations must fully interpret the HTML DOM and return\na deterministic, structured output.\n\nReturns:\n Parsed representation of type `T`." + "signature": "", + "docstring": "Fully parse the HTML content into structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the HTML DOM and return a\n deterministic, structured output." }, "parse_div": { "name": "parse_div", "kind": "function", "path": "omniread.html.parser.HTMLParser.parse_div", - "signature": "", - "docstring": "Extract normalized text from a `
` element.\n\nArgs:\n div: BeautifulSoup tag representing a `
`.\n separator: String used to separate text nodes.\n\nReturns:\n Flattened, whitespace-normalized text content." + "signature": "", + "docstring": "Extract normalized text from a `
` element.\n\nArgs:\n div (Tag):\n BeautifulSoup tag representing a `
`.\n separator (str, optional):\n String used to separate text nodes.\n\nReturns:\n str:\n Flattened, whitespace-normalized text content." }, "parse_link": { "name": "parse_link", "kind": "function", "path": "omniread.html.parser.HTMLParser.parse_link", - "signature": "", - "docstring": "Extract the hyperlink reference from an `` element.\n\nArgs:\n a: BeautifulSoup tag representing an anchor.\n\nReturns:\n The value of the `href` attribute, or None if absent." + "signature": "", + "docstring": "Extract the hyperlink reference from an `` element.\n\nArgs:\n a (Tag):\n BeautifulSoup tag representing an anchor.\n\nReturns:\n Optional[str]:\n The value of the `href` attribute, or None if absent." }, "parse_table": { "name": "parse_table", "kind": "function", "path": "omniread.html.parser.HTMLParser.parse_table", - "signature": "", - "docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table: BeautifulSoup tag representing a `
`.\n\nReturns:\n A list of rows, where each row is a list of cell text values." + "signature": "", + "docstring": "Parse an HTML table into a 2D list of strings.\n\nArgs:\n table (Tag):\n BeautifulSoup tag representing a `
`.\n\nReturns:\n list[list[str]]:\n A list of rows, where each row is a list of cell text values." }, "parse_meta": { "name": "parse_meta", "kind": "function", "path": "omniread.html.parser.HTMLParser.parse_meta", - "signature": "", - "docstring": "Extract high-level metadata from the HTML document.\n\nThis includes:\n- Document title\n- `` tag name/property → content mappings\n\nReturns:\n Dictionary containing extracted metadata." + "signature": "", + "docstring": "Extract high-level metadata from the HTML document.\n\nReturns:\n dict[str, Any]:\n Dictionary containing extracted metadata.\n\nNotes:\n **Responsibilities:**\n\n - Extract high-level metadata from the HTML document.\n - This includes: Document title, `` tag name/property to\n content mappings." } } }, @@ -1061,7 +1061,7 @@ "kind": "module", "path": "omniread.html.scraper", "signature": null, - "docstring": "HTML scraping implementation for OmniRead.\n\nThis module provides an HTTP-based scraper for retrieving HTML documents.\nIt implements the core `BaseScraper` contract using `httpx` as the transport\nlayer.\n\nThis scraper is responsible for:\n- Fetching raw HTML bytes over HTTP(S)\n- Validating response content type\n- Attaching HTTP metadata to the returned content\n\nThis scraper is not responsible for:\n- Parsing or interpreting HTML\n- Retrying failed requests\n- Managing crawl policies or rate limiting", + "docstring": "# Summary\n\nHTML scraping implementation for OmniRead.\n\nThis module provides an HTTP-based scraper for retrieving HTML documents.\nIt implements the core `BaseScraper` contract using `httpx` as the transport\nlayer.\n\nThis scraper is responsible for:\n\n- Fetching raw HTML bytes over HTTP(S)\n- Validating response content type\n- Attaching HTTP metadata to the returned content\n\nThis scraper is not responsible for:\n\n- Parsing or interpreting HTML\n- Retrying failed requests\n- Managing crawl policies or rate limiting", "members": { "httpx": { "name": "httpx", @@ -1096,35 +1096,35 @@ "kind": "class", "path": "omniread.html.scraper.Content", "signature": "", - "docstring": "Normalized representation of extracted content.\n\nA `Content` instance represents a raw content payload along with minimal\ncontextual metadata describing its origin and type.\n\nThis class is the **primary exchange format** between:\n- Scrapers\n- Parsers\n- Downstream consumers\n\nAttributes:\n raw: Raw content bytes as retrieved from the source.\n source: Identifier of the content origin (URL, file path, or logical name).\n content_type: Optional MIME type of the content, if known.\n metadata: Optional, implementation-defined metadata associated with\n the content (e.g., headers, encoding hints, extraction notes).", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.html.scraper.Content.raw", "signature": "", - "docstring": null + "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.html.scraper.Content.source", "signature": "", - "docstring": null + "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.html.scraper.Content.content_type", "signature": "", - "docstring": null + "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.html.scraper.Content.metadata", "signature": "", - "docstring": null + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } }, @@ -1133,7 +1133,7 @@ "kind": "class", "path": "omniread.html.scraper.ContentType", "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nThis enum represents the declared or inferred media type of the content\nsource. It is primarily used for routing content to the appropriate\nparser or downstream consumer.", + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", @@ -1170,14 +1170,14 @@ "kind": "class", "path": "omniread.html.scraper.BaseScraper", "signature": "", - "docstring": "Base interface for all scrapers.\n\nA scraper is responsible ONLY for fetching raw content\n(bytes) from a source. It must not interpret or parse it.\n\nA scraper is a **stateless acquisition component** that retrieves raw\ncontent from a source and returns it as a `Content` object.\n\nScrapers define *how content is obtained*, not *what the content means*.\n\nImplementations may vary in:\n- Transport mechanism (HTTP, filesystem, cloud storage)\n- Authentication strategy\n- Retry and backoff behavior\n\nImplementations must not:\n- Parse content\n- Modify content semantics\n- Couple scraping logic to a specific parser", + "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.html.scraper.BaseScraper.fetch", "signature": "", - "docstring": "Fetch raw content from the given source.\n\nImplementations must retrieve the content referenced by `source`\nand return it as raw bytes wrapped in a `Content` object.\n\nArgs:\n source: Location identifier (URL, file path, S3 URI, etc.)\n metadata: Optional hints for the scraper (headers, auth, etc.)\n\nReturns:\n Content object containing raw bytes and metadata.\n - Raw content bytes\n - Source identifier\n - Optional metadata\n\nRaises:\n Exception: Retrieval-specific errors as defined by the implementation." + "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." } } }, @@ -1185,8 +1185,8 @@ "name": "HTMLScraper", "kind": "class", "path": "omniread.html.scraper.HTMLScraper", - "signature": "", - "docstring": "Base HTML scraper using httpx.\n\nThis scraper retrieves HTML documents over HTTP(S) and returns them\nas raw content wrapped in a `Content` object.\n\nFetches raw bytes and metadata only.\nThe scraper:\n- Uses `httpx.Client` for HTTP requests\n- Enforces an HTML content type\n- Preserves HTTP response metadata\n\nThe scraper does not:\n- Parse HTML\n- Perform retries or backoff\n- Handle non-HTML responses", + "signature": "", + "docstring": "Base HTML scraper using `httpx`.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns\n them as raw content wrapped in a `Content` object.\n - Fetches raw bytes and metadata only.\n - The scraper uses `httpx.Client` for HTTP requests, enforces an\n HTML content type, and preserves HTTP response metadata.\n\n **Constraints:**\n\n - The scraper does not: Parse HTML, perform retries or backoff,\n handle non-HTML responses.", "members": { "content_type": { "name": "content_type", @@ -1199,15 +1199,15 @@ "name": "validate_content_type", "kind": "function", "path": "omniread.html.scraper.HTMLScraper.validate_content_type", - "signature": "", - "docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response: HTTP response returned by `httpx`.\n\nRaises:\n ValueError: If the `Content-Type` header is missing or does not\n indicate HTML content." + "signature": "", + "docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content." }, "fetch": { "name": "fetch", "kind": "function", "path": "omniread.html.scraper.HTMLScraper.fetch", - "signature": "", - "docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source: URL of the HTML document.\n metadata: Optional metadata to be merged into the returned content.\n\nReturns:\n A `Content` instance containing:\n - Raw HTML bytes\n - Source URL\n - HTML content type\n - HTTP response metadata\n\nRaises:\n httpx.HTTPError: If the HTTP request fails.\n ValueError: If the response is not valid HTML." + "signature": "", + "docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML." } } } @@ -1220,21 +1220,21 @@ "kind": "module", "path": "omniread.pdf", "signature": null, - "docstring": "PDF format implementation for OmniRead.\n\nThis package provides **PDF-specific implementations** of the core OmniRead\ncontracts defined in `omniread.core`.\n\nUnlike HTML, PDF handling requires an explicit client layer for document\naccess. This package therefore includes:\n- PDF clients for acquiring raw PDF data\n- PDF scrapers that coordinate client access\n- PDF parsers that extract structured content from PDF binaries\n\nPublic exports from this package represent the supported PDF pipeline\nand are safe for consumers to import directly when working with PDFs.", + "docstring": "# Summary\n\nPDF format implementation for OmniRead.\n\nThis package provides **PDF-specific implementations** of the core OmniRead\ncontracts defined in `omniread.core`.\n\nUnlike HTML, PDF handling requires an explicit client layer for document\naccess. This package therefore includes:\n\n- PDF clients for acquiring raw PDF data.\n- PDF scrapers that coordinate client access.\n- PDF parsers that extract structured content from PDF binaries.\n\nPublic exports from this package represent the supported PDF pipeline\nand are safe for consumers to import directly when working with PDFs.\n\n---\n\n# Public API\n\n- `FileSystemPDFClient`\n- `PDFScraper`\n- `PDFParser`\n\n---", "members": { "FileSystemPDFClient": { "name": "FileSystemPDFClient", "kind": "class", "path": "omniread.pdf.FileSystemPDFClient", "signature": "", - "docstring": "PDF client that reads from the local filesystem.\n\nThis client reads PDF files directly from the disk and returns their raw\nbinary contents.", + "docstring": "PDF client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads PDF files directly from the disk and returns\n their raw binary contents.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.FileSystemPDFClient.fetch", "signature": "", - "docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path: Filesystem path to the PDF file.\n\nReturns:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError: If the path does not exist.\n ValueError: If the path exists but is not a file." + "docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the PDF file.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." } } }, @@ -1243,14 +1243,14 @@ "kind": "class", "path": "omniread.pdf.PDFScraper", "signature": "", - "docstring": "Scraper for PDF sources.\n\nDelegates byte retrieval to a PDF client and normalizes\noutput into Content.\n\nThe scraper:\n- Does not perform parsing or interpretation\n- Does not assume a specific storage backend\n- Preserves caller-provided metadata", + "docstring": "Scraper for PDF sources.\n\nNotes:\n **Responsibilities:**\n\n - Delegates byte retrieval to a PDF client and normalizes output\n into `Content`.\n - Preserves caller-provided metadata.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.PDFScraper.fetch", "signature": "", - "docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source: Identifier of the PDF source as understood by the\n configured PDF client.\n metadata: Optional metadata to attach to the returned content.\n\nReturns:\n A `Content` instance containing:\n - Raw PDF bytes\n - Source identifier\n - PDF content type\n - Optional metadata\n\nRaises:\n Exception: Retrieval-specific errors raised by the PDF client." + "docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client." } } }, @@ -1259,7 +1259,7 @@ "kind": "class", "path": "omniread.pdf.PDFParser", "signature": "", - "docstring": "Base PDF parser.\n\nThis class enforces PDF content-type compatibility and provides the\nextension point for implementing concrete PDF parsing strategies.\n\nConcrete implementations must define:\n- Define the output type `T`\n- Implement the `parse()` method", + "docstring": "Base PDF parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces PDF content-type compatibility and provides\n the extension point for implementing concrete PDF parsing strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", "members": { "supported_types": { "name": "supported_types", @@ -1273,7 +1273,7 @@ "kind": "function", "path": "omniread.pdf.PDFParser.parse", "signature": "", - "docstring": "Parse PDF content into a structured output.\n\nImplementations must fully interpret the PDF binary payload and\nreturn a deterministic, structured output.\n\nReturns:\n Parsed representation of type `T`.\n\nRaises:\n Exception: Parsing-specific errors as defined by the implementation." + "docstring": "Parse PDF content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the PDF binary payload and\n return a deterministic, structured output." } } }, @@ -1282,7 +1282,7 @@ "kind": "module", "path": "omniread.pdf.client", "signature": null, - "docstring": "PDF client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw PDF\nbytes from a concrete backing store.\n\nClients provide low-level access to PDF binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems", + "docstring": "# Summary\n\nPDF client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw PDF\nbytes from a concrete backing store.\n\nClients provide low-level access to PDF binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems", "members": { "Any": { "name": "Any", @@ -1316,15 +1316,15 @@ "name": "BasePDFClient", "kind": "class", "path": "omniread.pdf.client.BasePDFClient", - "signature": "", - "docstring": "Abstract client responsible for retrieving PDF bytes\nfrom a specific backing store (filesystem, S3, FTP, etc.).\n\nImplementations must:\n- Accept a source identifier appropriate to the backing store\n- Return the full PDF binary payload\n- Raise retrieval-specific errors on failure", + "signature": "", + "docstring": "Abstract client responsible for retrieving PDF bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full PDF binary payload.\n - Raise retrieval-specific errors on failure.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.client.BasePDFClient.fetch", - "signature": "", - "docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source: Identifier of the PDF location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n Raw PDF bytes.\n\nRaises:\n Exception: Retrieval-specific errors defined by the implementation." + "signature": "", + "docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF location, such as a file path, object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." } } }, @@ -1332,15 +1332,15 @@ "name": "FileSystemPDFClient", "kind": "class", "path": "omniread.pdf.client.FileSystemPDFClient", - "signature": "", - "docstring": "PDF client that reads from the local filesystem.\n\nThis client reads PDF files directly from the disk and returns their raw\nbinary contents.", + "signature": "", + "docstring": "PDF client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads PDF files directly from the disk and returns\n their raw binary contents.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.client.FileSystemPDFClient.fetch", - "signature": "", - "docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path: Filesystem path to the PDF file.\n\nReturns:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError: If the path does not exist.\n ValueError: If the path exists but is not a file." + "signature": "", + "docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the PDF file.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." } } } @@ -1351,7 +1351,7 @@ "kind": "module", "path": "omniread.pdf.parser", "signature": null, - "docstring": "PDF parser base implementations for OmniRead.\n\nThis module defines the **PDF-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for PDF content.\n\nPDF parsers are responsible for interpreting binary PDF data and producing\nstructured representations suitable for downstream consumption.", + "docstring": "# Summary\n\nPDF parser base implementations for OmniRead.\n\nThis module defines the **PDF-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for PDF content.\n\nPDF parsers are responsible for interpreting binary PDF data and producing\nstructured representations suitable for downstream consumption.", "members": { "Generic": { "name": "Generic", @@ -1379,7 +1379,7 @@ "kind": "class", "path": "omniread.pdf.parser.ContentType", "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nThis enum represents the declared or inferred media type of the content\nsource. It is primarily used for routing content to the appropriate\nparser or downstream consumer.", + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", @@ -1416,14 +1416,14 @@ "kind": "class", "path": "omniread.pdf.parser.BaseParser", "signature": "", - "docstring": "Base interface for all parsers.\n\nA parser is a self-contained object that owns the Content\nit is responsible for interpreting.\n\nImplementations must:\n- Declare supported content types via `supported_types`\n- Raise parsing-specific exceptions from `parse()`\n- Remain deterministic for a given input\n\nConsumers may rely on:\n- Early validation of content compatibility\n- Type-stable return values from `parse()`", + "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", "members": { "supported_types": { "name": "supported_types", "kind": "attribute", "path": "omniread.pdf.parser.BaseParser.supported_types", "signature": "", - "docstring": "Set of content types supported by this parser.\n\nAn empty set indicates that the parser is content-type agnostic." + "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." }, "content": { "name": "content", @@ -1437,14 +1437,14 @@ "kind": "function", "path": "omniread.pdf.parser.BaseParser.parse", "signature": "", - "docstring": "Parse the owned content into structured output.\n\nImplementations must fully consume the provided content and\nreturn a deterministic, structured output.\n\nReturns:\n Parsed, structured representation.\n\nRaises:\n Exception: Parsing-specific errors as defined by the implementation." + "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." }, "supports": { "name": "supports", "kind": "function", "path": "omniread.pdf.parser.BaseParser.supports", "signature": "", - "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n True if the content type is supported; False otherwise." + "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." } } }, @@ -1459,8 +1459,8 @@ "name": "PDFParser", "kind": "class", "path": "omniread.pdf.parser.PDFParser", - "signature": "", - "docstring": "Base PDF parser.\n\nThis class enforces PDF content-type compatibility and provides the\nextension point for implementing concrete PDF parsing strategies.\n\nConcrete implementations must define:\n- Define the output type `T`\n- Implement the `parse()` method", + "signature": "", + "docstring": "Base PDF parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces PDF content-type compatibility and provides\n the extension point for implementing concrete PDF parsing strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", "members": { "supported_types": { "name": "supported_types", @@ -1473,8 +1473,8 @@ "name": "parse", "kind": "function", "path": "omniread.pdf.parser.PDFParser.parse", - "signature": "", - "docstring": "Parse PDF content into a structured output.\n\nImplementations must fully interpret the PDF binary payload and\nreturn a deterministic, structured output.\n\nReturns:\n Parsed representation of type `T`.\n\nRaises:\n Exception: Parsing-specific errors as defined by the implementation." + "signature": "", + "docstring": "Parse PDF content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the PDF binary payload and\n return a deterministic, structured output." } } } @@ -1485,7 +1485,7 @@ "kind": "module", "path": "omniread.pdf.scraper", "signature": null, - "docstring": "PDF scraping implementation for OmniRead.\n\nThis module provides a PDF-specific scraper that coordinates PDF byte\nretrieval via a client and normalizes the result into a `Content` object.\n\nThe scraper implements the core `BaseScraper` contract while delegating\nall storage and access concerns to a `BasePDFClient` implementation.", + "docstring": "# Summary\n\nPDF scraping implementation for OmniRead.\n\nThis module provides a PDF-specific scraper that coordinates PDF byte\nretrieval via a client and normalizes the result into a `Content` object.\n\nThe scraper implements the core `BaseScraper` contract while delegating\nall storage and access concerns to a `BasePDFClient` implementation.", "members": { "Any": { "name": "Any", @@ -1513,35 +1513,35 @@ "kind": "class", "path": "omniread.pdf.scraper.Content", "signature": "", - "docstring": "Normalized representation of extracted content.\n\nA `Content` instance represents a raw content payload along with minimal\ncontextual metadata describing its origin and type.\n\nThis class is the **primary exchange format** between:\n- Scrapers\n- Parsers\n- Downstream consumers\n\nAttributes:\n raw: Raw content bytes as retrieved from the source.\n source: Identifier of the content origin (URL, file path, or logical name).\n content_type: Optional MIME type of the content, if known.\n metadata: Optional, implementation-defined metadata associated with\n the content (e.g., headers, encoding hints, extraction notes).", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.pdf.scraper.Content.raw", "signature": "", - "docstring": null + "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.pdf.scraper.Content.source", "signature": "", - "docstring": null + "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.pdf.scraper.Content.content_type", "signature": "", - "docstring": null + "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.pdf.scraper.Content.metadata", "signature": "", - "docstring": null + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } }, @@ -1550,7 +1550,7 @@ "kind": "class", "path": "omniread.pdf.scraper.ContentType", "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nThis enum represents the declared or inferred media type of the content\nsource. It is primarily used for routing content to the appropriate\nparser or downstream consumer.", + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", @@ -1587,14 +1587,14 @@ "kind": "class", "path": "omniread.pdf.scraper.BaseScraper", "signature": "", - "docstring": "Base interface for all scrapers.\n\nA scraper is responsible ONLY for fetching raw content\n(bytes) from a source. It must not interpret or parse it.\n\nA scraper is a **stateless acquisition component** that retrieves raw\ncontent from a source and returns it as a `Content` object.\n\nScrapers define *how content is obtained*, not *what the content means*.\n\nImplementations may vary in:\n- Transport mechanism (HTTP, filesystem, cloud storage)\n- Authentication strategy\n- Retry and backoff behavior\n\nImplementations must not:\n- Parse content\n- Modify content semantics\n- Couple scraping logic to a specific parser", + "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.scraper.BaseScraper.fetch", "signature": "", - "docstring": "Fetch raw content from the given source.\n\nImplementations must retrieve the content referenced by `source`\nand return it as raw bytes wrapped in a `Content` object.\n\nArgs:\n source: Location identifier (URL, file path, S3 URI, etc.)\n metadata: Optional hints for the scraper (headers, auth, etc.)\n\nReturns:\n Content object containing raw bytes and metadata.\n - Raw content bytes\n - Source identifier\n - Optional metadata\n\nRaises:\n Exception: Retrieval-specific errors as defined by the implementation." + "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." } } }, @@ -1603,14 +1603,14 @@ "kind": "class", "path": "omniread.pdf.scraper.BasePDFClient", "signature": "", - "docstring": "Abstract client responsible for retrieving PDF bytes\nfrom a specific backing store (filesystem, S3, FTP, etc.).\n\nImplementations must:\n- Accept a source identifier appropriate to the backing store\n- Return the full PDF binary payload\n- Raise retrieval-specific errors on failure", + "docstring": "Abstract client responsible for retrieving PDF bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full PDF binary payload.\n - Raise retrieval-specific errors on failure.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.scraper.BasePDFClient.fetch", "signature": "", - "docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source: Identifier of the PDF location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n Raw PDF bytes.\n\nRaises:\n Exception: Retrieval-specific errors defined by the implementation." + "docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF location, such as a file path, object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." } } }, @@ -1618,15 +1618,15 @@ "name": "PDFScraper", "kind": "class", "path": "omniread.pdf.scraper.PDFScraper", - "signature": "", - "docstring": "Scraper for PDF sources.\n\nDelegates byte retrieval to a PDF client and normalizes\noutput into Content.\n\nThe scraper:\n- Does not perform parsing or interpretation\n- Does not assume a specific storage backend\n- Preserves caller-provided metadata", + "signature": "", + "docstring": "Scraper for PDF sources.\n\nNotes:\n **Responsibilities:**\n\n - Delegates byte retrieval to a PDF client and normalizes output\n into `Content`.\n - Preserves caller-provided metadata.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.scraper.PDFScraper.fetch", - "signature": "", - "docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source: Identifier of the PDF source as understood by the\n configured PDF client.\n metadata: Optional metadata to attach to the returned content.\n\nReturns:\n A `Content` instance containing:\n - Raw PDF bytes\n - Source identifier\n - PDF content type\n - Optional metadata\n\nRaises:\n Exception: Retrieval-specific errors raised by the PDF client." + "signature": "", + "docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client." } } } diff --git a/mcp_docs/modules/omniread.pdf.client.json b/mcp_docs/modules/omniread.pdf.client.json index 127ef5a..d7e0484 100644 --- a/mcp_docs/modules/omniread.pdf.client.json +++ b/mcp_docs/modules/omniread.pdf.client.json @@ -2,7 +2,7 @@ "module": "omniread.pdf.client", "content": { "path": "omniread.pdf.client", - "docstring": "PDF client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw PDF\nbytes from a concrete backing store.\n\nClients provide low-level access to PDF binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems", + "docstring": "# Summary\n\nPDF client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw PDF\nbytes from a concrete backing store.\n\nClients provide low-level access to PDF binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems", "objects": { "Any": { "name": "Any", @@ -36,15 +36,15 @@ "name": "BasePDFClient", "kind": "class", "path": "omniread.pdf.client.BasePDFClient", - "signature": "", - "docstring": "Abstract client responsible for retrieving PDF bytes\nfrom a specific backing store (filesystem, S3, FTP, etc.).\n\nImplementations must:\n- Accept a source identifier appropriate to the backing store\n- Return the full PDF binary payload\n- Raise retrieval-specific errors on failure", + "signature": "", + "docstring": "Abstract client responsible for retrieving PDF bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full PDF binary payload.\n - Raise retrieval-specific errors on failure.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.client.BasePDFClient.fetch", - "signature": "", - "docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source: Identifier of the PDF location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n Raw PDF bytes.\n\nRaises:\n Exception: Retrieval-specific errors defined by the implementation." + "signature": "", + "docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF location, such as a file path, object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." } } }, @@ -52,15 +52,15 @@ "name": "FileSystemPDFClient", "kind": "class", "path": "omniread.pdf.client.FileSystemPDFClient", - "signature": "", - "docstring": "PDF client that reads from the local filesystem.\n\nThis client reads PDF files directly from the disk and returns their raw\nbinary contents.", + "signature": "", + "docstring": "PDF client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads PDF files directly from the disk and returns\n their raw binary contents.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.client.FileSystemPDFClient.fetch", - "signature": "", - "docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path: Filesystem path to the PDF file.\n\nReturns:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError: If the path does not exist.\n ValueError: If the path exists but is not a file." + "signature": "", + "docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the PDF file.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." } } } diff --git a/mcp_docs/modules/omniread.pdf.json b/mcp_docs/modules/omniread.pdf.json index 9067c6d..ca41c24 100644 --- a/mcp_docs/modules/omniread.pdf.json +++ b/mcp_docs/modules/omniread.pdf.json @@ -2,21 +2,21 @@ "module": "omniread.pdf", "content": { "path": "omniread.pdf", - "docstring": "PDF format implementation for OmniRead.\n\nThis package provides **PDF-specific implementations** of the core OmniRead\ncontracts defined in `omniread.core`.\n\nUnlike HTML, PDF handling requires an explicit client layer for document\naccess. This package therefore includes:\n- PDF clients for acquiring raw PDF data\n- PDF scrapers that coordinate client access\n- PDF parsers that extract structured content from PDF binaries\n\nPublic exports from this package represent the supported PDF pipeline\nand are safe for consumers to import directly when working with PDFs.", + "docstring": "# Summary\n\nPDF format implementation for OmniRead.\n\nThis package provides **PDF-specific implementations** of the core OmniRead\ncontracts defined in `omniread.core`.\n\nUnlike HTML, PDF handling requires an explicit client layer for document\naccess. This package therefore includes:\n\n- PDF clients for acquiring raw PDF data.\n- PDF scrapers that coordinate client access.\n- PDF parsers that extract structured content from PDF binaries.\n\nPublic exports from this package represent the supported PDF pipeline\nand are safe for consumers to import directly when working with PDFs.\n\n---\n\n# Public API\n\n- `FileSystemPDFClient`\n- `PDFScraper`\n- `PDFParser`\n\n---", "objects": { "FileSystemPDFClient": { "name": "FileSystemPDFClient", "kind": "class", "path": "omniread.pdf.FileSystemPDFClient", "signature": "", - "docstring": "PDF client that reads from the local filesystem.\n\nThis client reads PDF files directly from the disk and returns their raw\nbinary contents.", + "docstring": "PDF client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads PDF files directly from the disk and returns\n their raw binary contents.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.FileSystemPDFClient.fetch", "signature": "", - "docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path: Filesystem path to the PDF file.\n\nReturns:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError: If the path does not exist.\n ValueError: If the path exists but is not a file." + "docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the PDF file.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." } } }, @@ -25,14 +25,14 @@ "kind": "class", "path": "omniread.pdf.PDFScraper", "signature": "", - "docstring": "Scraper for PDF sources.\n\nDelegates byte retrieval to a PDF client and normalizes\noutput into Content.\n\nThe scraper:\n- Does not perform parsing or interpretation\n- Does not assume a specific storage backend\n- Preserves caller-provided metadata", + "docstring": "Scraper for PDF sources.\n\nNotes:\n **Responsibilities:**\n\n - Delegates byte retrieval to a PDF client and normalizes output\n into `Content`.\n - Preserves caller-provided metadata.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.PDFScraper.fetch", "signature": "", - "docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source: Identifier of the PDF source as understood by the\n configured PDF client.\n metadata: Optional metadata to attach to the returned content.\n\nReturns:\n A `Content` instance containing:\n - Raw PDF bytes\n - Source identifier\n - PDF content type\n - Optional metadata\n\nRaises:\n Exception: Retrieval-specific errors raised by the PDF client." + "docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client." } } }, @@ -41,7 +41,7 @@ "kind": "class", "path": "omniread.pdf.PDFParser", "signature": "", - "docstring": "Base PDF parser.\n\nThis class enforces PDF content-type compatibility and provides the\nextension point for implementing concrete PDF parsing strategies.\n\nConcrete implementations must define:\n- Define the output type `T`\n- Implement the `parse()` method", + "docstring": "Base PDF parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces PDF content-type compatibility and provides\n the extension point for implementing concrete PDF parsing strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", "members": { "supported_types": { "name": "supported_types", @@ -55,7 +55,7 @@ "kind": "function", "path": "omniread.pdf.PDFParser.parse", "signature": "", - "docstring": "Parse PDF content into a structured output.\n\nImplementations must fully interpret the PDF binary payload and\nreturn a deterministic, structured output.\n\nReturns:\n Parsed representation of type `T`.\n\nRaises:\n Exception: Parsing-specific errors as defined by the implementation." + "docstring": "Parse PDF content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the PDF binary payload and\n return a deterministic, structured output." } } }, @@ -64,7 +64,7 @@ "kind": "module", "path": "omniread.pdf.client", "signature": null, - "docstring": "PDF client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw PDF\nbytes from a concrete backing store.\n\nClients provide low-level access to PDF binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems", + "docstring": "# Summary\n\nPDF client abstractions for OmniRead.\n\nThis module defines the **client layer** responsible for retrieving raw PDF\nbytes from a concrete backing store.\n\nClients provide low-level access to PDF binaries and are intentionally\ndecoupled from scraping and parsing logic. They do not perform validation,\ninterpretation, or content extraction.\n\nTypical backing stores include:\n\n- Local filesystems\n- Object storage (S3, GCS, etc.)\n- Network file systems", "members": { "Any": { "name": "Any", @@ -98,15 +98,15 @@ "name": "BasePDFClient", "kind": "class", "path": "omniread.pdf.client.BasePDFClient", - "signature": "", - "docstring": "Abstract client responsible for retrieving PDF bytes\nfrom a specific backing store (filesystem, S3, FTP, etc.).\n\nImplementations must:\n- Accept a source identifier appropriate to the backing store\n- Return the full PDF binary payload\n- Raise retrieval-specific errors on failure", + "signature": "", + "docstring": "Abstract client responsible for retrieving PDF bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full PDF binary payload.\n - Raise retrieval-specific errors on failure.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.client.BasePDFClient.fetch", - "signature": "", - "docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source: Identifier of the PDF location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n Raw PDF bytes.\n\nRaises:\n Exception: Retrieval-specific errors defined by the implementation." + "signature": "", + "docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF location, such as a file path, object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." } } }, @@ -114,15 +114,15 @@ "name": "FileSystemPDFClient", "kind": "class", "path": "omniread.pdf.client.FileSystemPDFClient", - "signature": "", - "docstring": "PDF client that reads from the local filesystem.\n\nThis client reads PDF files directly from the disk and returns their raw\nbinary contents.", + "signature": "", + "docstring": "PDF client that reads from the local filesystem.\n\nNotes:\n **Guarantees:**\n\n - This client reads PDF files directly from the disk and returns\n their raw binary contents.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.client.FileSystemPDFClient.fetch", - "signature": "", - "docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path: Filesystem path to the PDF file.\n\nReturns:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError: If the path does not exist.\n ValueError: If the path exists but is not a file." + "signature": "", + "docstring": "Read a PDF file from the local filesystem.\n\nArgs:\n path (Path):\n Filesystem path to the PDF file.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n FileNotFoundError:\n If the path does not exist.\n ValueError:\n If the path exists but is not a file." } } } @@ -133,7 +133,7 @@ "kind": "module", "path": "omniread.pdf.parser", "signature": null, - "docstring": "PDF parser base implementations for OmniRead.\n\nThis module defines the **PDF-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for PDF content.\n\nPDF parsers are responsible for interpreting binary PDF data and producing\nstructured representations suitable for downstream consumption.", + "docstring": "# Summary\n\nPDF parser base implementations for OmniRead.\n\nThis module defines the **PDF-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for PDF content.\n\nPDF parsers are responsible for interpreting binary PDF data and producing\nstructured representations suitable for downstream consumption.", "members": { "Generic": { "name": "Generic", @@ -161,7 +161,7 @@ "kind": "class", "path": "omniread.pdf.parser.ContentType", "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nThis enum represents the declared or inferred media type of the content\nsource. It is primarily used for routing content to the appropriate\nparser or downstream consumer.", + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", @@ -198,14 +198,14 @@ "kind": "class", "path": "omniread.pdf.parser.BaseParser", "signature": "", - "docstring": "Base interface for all parsers.\n\nA parser is a self-contained object that owns the Content\nit is responsible for interpreting.\n\nImplementations must:\n- Declare supported content types via `supported_types`\n- Raise parsing-specific exceptions from `parse()`\n- Remain deterministic for a given input\n\nConsumers may rely on:\n- Early validation of content compatibility\n- Type-stable return values from `parse()`", + "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", "members": { "supported_types": { "name": "supported_types", "kind": "attribute", "path": "omniread.pdf.parser.BaseParser.supported_types", "signature": "", - "docstring": "Set of content types supported by this parser.\n\nAn empty set indicates that the parser is content-type agnostic." + "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." }, "content": { "name": "content", @@ -219,14 +219,14 @@ "kind": "function", "path": "omniread.pdf.parser.BaseParser.parse", "signature": "", - "docstring": "Parse the owned content into structured output.\n\nImplementations must fully consume the provided content and\nreturn a deterministic, structured output.\n\nReturns:\n Parsed, structured representation.\n\nRaises:\n Exception: Parsing-specific errors as defined by the implementation." + "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." }, "supports": { "name": "supports", "kind": "function", "path": "omniread.pdf.parser.BaseParser.supports", "signature": "", - "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n True if the content type is supported; False otherwise." + "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." } } }, @@ -241,8 +241,8 @@ "name": "PDFParser", "kind": "class", "path": "omniread.pdf.parser.PDFParser", - "signature": "", - "docstring": "Base PDF parser.\n\nThis class enforces PDF content-type compatibility and provides the\nextension point for implementing concrete PDF parsing strategies.\n\nConcrete implementations must define:\n- Define the output type `T`\n- Implement the `parse()` method", + "signature": "", + "docstring": "Base PDF parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces PDF content-type compatibility and provides\n the extension point for implementing concrete PDF parsing strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", "members": { "supported_types": { "name": "supported_types", @@ -255,8 +255,8 @@ "name": "parse", "kind": "function", "path": "omniread.pdf.parser.PDFParser.parse", - "signature": "", - "docstring": "Parse PDF content into a structured output.\n\nImplementations must fully interpret the PDF binary payload and\nreturn a deterministic, structured output.\n\nReturns:\n Parsed representation of type `T`.\n\nRaises:\n Exception: Parsing-specific errors as defined by the implementation." + "signature": "", + "docstring": "Parse PDF content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the PDF binary payload and\n return a deterministic, structured output." } } } @@ -267,7 +267,7 @@ "kind": "module", "path": "omniread.pdf.scraper", "signature": null, - "docstring": "PDF scraping implementation for OmniRead.\n\nThis module provides a PDF-specific scraper that coordinates PDF byte\nretrieval via a client and normalizes the result into a `Content` object.\n\nThe scraper implements the core `BaseScraper` contract while delegating\nall storage and access concerns to a `BasePDFClient` implementation.", + "docstring": "# Summary\n\nPDF scraping implementation for OmniRead.\n\nThis module provides a PDF-specific scraper that coordinates PDF byte\nretrieval via a client and normalizes the result into a `Content` object.\n\nThe scraper implements the core `BaseScraper` contract while delegating\nall storage and access concerns to a `BasePDFClient` implementation.", "members": { "Any": { "name": "Any", @@ -295,35 +295,35 @@ "kind": "class", "path": "omniread.pdf.scraper.Content", "signature": "", - "docstring": "Normalized representation of extracted content.\n\nA `Content` instance represents a raw content payload along with minimal\ncontextual metadata describing its origin and type.\n\nThis class is the **primary exchange format** between:\n- Scrapers\n- Parsers\n- Downstream consumers\n\nAttributes:\n raw: Raw content bytes as retrieved from the source.\n source: Identifier of the content origin (URL, file path, or logical name).\n content_type: Optional MIME type of the content, if known.\n metadata: Optional, implementation-defined metadata associated with\n the content (e.g., headers, encoding hints, extraction notes).", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.pdf.scraper.Content.raw", "signature": "", - "docstring": null + "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.pdf.scraper.Content.source", "signature": "", - "docstring": null + "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.pdf.scraper.Content.content_type", "signature": "", - "docstring": null + "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.pdf.scraper.Content.metadata", "signature": "", - "docstring": null + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } }, @@ -332,7 +332,7 @@ "kind": "class", "path": "omniread.pdf.scraper.ContentType", "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nThis enum represents the declared or inferred media type of the content\nsource. It is primarily used for routing content to the appropriate\nparser or downstream consumer.", + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", @@ -369,14 +369,14 @@ "kind": "class", "path": "omniread.pdf.scraper.BaseScraper", "signature": "", - "docstring": "Base interface for all scrapers.\n\nA scraper is responsible ONLY for fetching raw content\n(bytes) from a source. It must not interpret or parse it.\n\nA scraper is a **stateless acquisition component** that retrieves raw\ncontent from a source and returns it as a `Content` object.\n\nScrapers define *how content is obtained*, not *what the content means*.\n\nImplementations may vary in:\n- Transport mechanism (HTTP, filesystem, cloud storage)\n- Authentication strategy\n- Retry and backoff behavior\n\nImplementations must not:\n- Parse content\n- Modify content semantics\n- Couple scraping logic to a specific parser", + "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.scraper.BaseScraper.fetch", "signature": "", - "docstring": "Fetch raw content from the given source.\n\nImplementations must retrieve the content referenced by `source`\nand return it as raw bytes wrapped in a `Content` object.\n\nArgs:\n source: Location identifier (URL, file path, S3 URI, etc.)\n metadata: Optional hints for the scraper (headers, auth, etc.)\n\nReturns:\n Content object containing raw bytes and metadata.\n - Raw content bytes\n - Source identifier\n - Optional metadata\n\nRaises:\n Exception: Retrieval-specific errors as defined by the implementation." + "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." } } }, @@ -385,14 +385,14 @@ "kind": "class", "path": "omniread.pdf.scraper.BasePDFClient", "signature": "", - "docstring": "Abstract client responsible for retrieving PDF bytes\nfrom a specific backing store (filesystem, S3, FTP, etc.).\n\nImplementations must:\n- Accept a source identifier appropriate to the backing store\n- Return the full PDF binary payload\n- Raise retrieval-specific errors on failure", + "docstring": "Abstract client responsible for retrieving PDF bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full PDF binary payload.\n - Raise retrieval-specific errors on failure.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.scraper.BasePDFClient.fetch", "signature": "", - "docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source: Identifier of the PDF location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n Raw PDF bytes.\n\nRaises:\n Exception: Retrieval-specific errors defined by the implementation." + "docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF location, such as a file path, object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." } } }, @@ -400,15 +400,15 @@ "name": "PDFScraper", "kind": "class", "path": "omniread.pdf.scraper.PDFScraper", - "signature": "", - "docstring": "Scraper for PDF sources.\n\nDelegates byte retrieval to a PDF client and normalizes\noutput into Content.\n\nThe scraper:\n- Does not perform parsing or interpretation\n- Does not assume a specific storage backend\n- Preserves caller-provided metadata", + "signature": "", + "docstring": "Scraper for PDF sources.\n\nNotes:\n **Responsibilities:**\n\n - Delegates byte retrieval to a PDF client and normalizes output\n into `Content`.\n - Preserves caller-provided metadata.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.scraper.PDFScraper.fetch", - "signature": "", - "docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source: Identifier of the PDF source as understood by the\n configured PDF client.\n metadata: Optional metadata to attach to the returned content.\n\nReturns:\n A `Content` instance containing:\n - Raw PDF bytes\n - Source identifier\n - PDF content type\n - Optional metadata\n\nRaises:\n Exception: Retrieval-specific errors raised by the PDF client." + "signature": "", + "docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client." } } } diff --git a/mcp_docs/modules/omniread.pdf.parser.json b/mcp_docs/modules/omniread.pdf.parser.json index 471c058..e4fccb5 100644 --- a/mcp_docs/modules/omniread.pdf.parser.json +++ b/mcp_docs/modules/omniread.pdf.parser.json @@ -2,7 +2,7 @@ "module": "omniread.pdf.parser", "content": { "path": "omniread.pdf.parser", - "docstring": "PDF parser base implementations for OmniRead.\n\nThis module defines the **PDF-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for PDF content.\n\nPDF parsers are responsible for interpreting binary PDF data and producing\nstructured representations suitable for downstream consumption.", + "docstring": "# Summary\n\nPDF parser base implementations for OmniRead.\n\nThis module defines the **PDF-specific parser contract**, extending the\nformat-agnostic `BaseParser` with constraints appropriate for PDF content.\n\nPDF parsers are responsible for interpreting binary PDF data and producing\nstructured representations suitable for downstream consumption.", "objects": { "Generic": { "name": "Generic", @@ -30,7 +30,7 @@ "kind": "class", "path": "omniread.pdf.parser.ContentType", "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nThis enum represents the declared or inferred media type of the content\nsource. It is primarily used for routing content to the appropriate\nparser or downstream consumer.", + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", @@ -67,14 +67,14 @@ "kind": "class", "path": "omniread.pdf.parser.BaseParser", "signature": "", - "docstring": "Base interface for all parsers.\n\nA parser is a self-contained object that owns the Content\nit is responsible for interpreting.\n\nImplementations must:\n- Declare supported content types via `supported_types`\n- Raise parsing-specific exceptions from `parse()`\n- Remain deterministic for a given input\n\nConsumers may rely on:\n- Early validation of content compatibility\n- Type-stable return values from `parse()`", + "docstring": "Base interface for all parsers.\n\nNotes:\n **Guarantees:**\n\n - A parser is a self-contained object that owns the `Content` it is\n responsible for interpreting.\n - Consumers may rely on early validation of content compatibility\n and type-stable return values from `parse()`.\n\n **Responsibilities:**\n\n - Implementations must declare supported content types via `supported_types`.\n - Implementations must raise parsing-specific exceptions from `parse()`.\n - Implementations must remain deterministic for a given input.", "members": { "supported_types": { "name": "supported_types", "kind": "attribute", "path": "omniread.pdf.parser.BaseParser.supported_types", "signature": "", - "docstring": "Set of content types supported by this parser.\n\nAn empty set indicates that the parser is content-type agnostic." + "docstring": "Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic." }, "content": { "name": "content", @@ -88,14 +88,14 @@ "kind": "function", "path": "omniread.pdf.parser.BaseParser.parse", "signature": "", - "docstring": "Parse the owned content into structured output.\n\nImplementations must fully consume the provided content and\nreturn a deterministic, structured output.\n\nReturns:\n Parsed, structured representation.\n\nRaises:\n Exception: Parsing-specific errors as defined by the implementation." + "docstring": "Parse the owned content into structured output.\n\nReturns:\n T:\n Parsed, structured representation.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully consume the provided content and\n return a deterministic, structured output." }, "supports": { "name": "supports", "kind": "function", "path": "omniread.pdf.parser.BaseParser.supports", "signature": "", - "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n True if the content type is supported; False otherwise." + "docstring": "Check whether this parser supports the content's type.\n\nReturns:\n bool:\n True if the content type is supported; False otherwise." } } }, @@ -110,8 +110,8 @@ "name": "PDFParser", "kind": "class", "path": "omniread.pdf.parser.PDFParser", - "signature": "", - "docstring": "Base PDF parser.\n\nThis class enforces PDF content-type compatibility and provides the\nextension point for implementing concrete PDF parsing strategies.\n\nConcrete implementations must define:\n- Define the output type `T`\n- Implement the `parse()` method", + "signature": "", + "docstring": "Base PDF parser.\n\nNotes:\n **Responsibilities:**\n\n - This class enforces PDF content-type compatibility and provides\n the extension point for implementing concrete PDF parsing strategies.\n\n **Constraints:**\n\n - Concrete implementations must define the output type `T` and\n implement the `parse()` method.", "members": { "supported_types": { "name": "supported_types", @@ -124,8 +124,8 @@ "name": "parse", "kind": "function", "path": "omniread.pdf.parser.PDFParser.parse", - "signature": "", - "docstring": "Parse PDF content into a structured output.\n\nImplementations must fully interpret the PDF binary payload and\nreturn a deterministic, structured output.\n\nReturns:\n Parsed representation of type `T`.\n\nRaises:\n Exception: Parsing-specific errors as defined by the implementation." + "signature": "", + "docstring": "Parse PDF content into a structured output.\n\nReturns:\n T:\n Parsed representation of type `T`.\n\nRaises:\n Exception:\n Parsing-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must fully interpret the PDF binary payload and\n return a deterministic, structured output." } } } diff --git a/mcp_docs/modules/omniread.pdf.scraper.json b/mcp_docs/modules/omniread.pdf.scraper.json index 3d5756f..23de5a1 100644 --- a/mcp_docs/modules/omniread.pdf.scraper.json +++ b/mcp_docs/modules/omniread.pdf.scraper.json @@ -2,7 +2,7 @@ "module": "omniread.pdf.scraper", "content": { "path": "omniread.pdf.scraper", - "docstring": "PDF scraping implementation for OmniRead.\n\nThis module provides a PDF-specific scraper that coordinates PDF byte\nretrieval via a client and normalizes the result into a `Content` object.\n\nThe scraper implements the core `BaseScraper` contract while delegating\nall storage and access concerns to a `BasePDFClient` implementation.", + "docstring": "# Summary\n\nPDF scraping implementation for OmniRead.\n\nThis module provides a PDF-specific scraper that coordinates PDF byte\nretrieval via a client and normalizes the result into a `Content` object.\n\nThe scraper implements the core `BaseScraper` contract while delegating\nall storage and access concerns to a `BasePDFClient` implementation.", "objects": { "Any": { "name": "Any", @@ -30,35 +30,35 @@ "kind": "class", "path": "omniread.pdf.scraper.Content", "signature": "", - "docstring": "Normalized representation of extracted content.\n\nA `Content` instance represents a raw content payload along with minimal\ncontextual metadata describing its origin and type.\n\nThis class is the **primary exchange format** between:\n- Scrapers\n- Parsers\n- Downstream consumers\n\nAttributes:\n raw: Raw content bytes as retrieved from the source.\n source: Identifier of the content origin (URL, file path, or logical name).\n content_type: Optional MIME type of the content, if known.\n metadata: Optional, implementation-defined metadata associated with\n the content (e.g., headers, encoding hints, extraction notes).", + "docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with\n minimal contextual metadata describing its origin and type.\n - This class is the primary exchange format between scrapers,\n parsers, and downstream consumers.", "members": { "raw": { "name": "raw", "kind": "attribute", "path": "omniread.pdf.scraper.Content.raw", "signature": "", - "docstring": null + "docstring": "Raw content bytes as retrieved from the source." }, "source": { "name": "source", "kind": "attribute", "path": "omniread.pdf.scraper.Content.source", "signature": "", - "docstring": null + "docstring": "Identifier of the content origin (URL, file path, or logical name)." }, "content_type": { "name": "content_type", "kind": "attribute", "path": "omniread.pdf.scraper.Content.content_type", "signature": "", - "docstring": null + "docstring": "Optional MIME type of the content, if known." }, "metadata": { "name": "metadata", "kind": "attribute", "path": "omniread.pdf.scraper.Content.metadata", "signature": "", - "docstring": null + "docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)." } } }, @@ -67,7 +67,7 @@ "kind": "class", "path": "omniread.pdf.scraper.ContentType", "signature": "", - "docstring": "Supported MIME types for extracted content.\n\nThis enum represents the declared or inferred media type of the content\nsource. It is primarily used for routing content to the appropriate\nparser or downstream consumer.", + "docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the\n content source.\n - It is primarily used for routing content to the appropriate\n parser or downstream consumer.", "members": { "HTML": { "name": "HTML", @@ -104,14 +104,14 @@ "kind": "class", "path": "omniread.pdf.scraper.BaseScraper", "signature": "", - "docstring": "Base interface for all scrapers.\n\nA scraper is responsible ONLY for fetching raw content\n(bytes) from a source. It must not interpret or parse it.\n\nA scraper is a **stateless acquisition component** that retrieves raw\ncontent from a source and returns it as a `Content` object.\n\nScrapers define *how content is obtained*, not *what the content means*.\n\nImplementations may vary in:\n- Transport mechanism (HTTP, filesystem, cloud storage)\n- Authentication strategy\n- Retry and backoff behavior\n\nImplementations must not:\n- Parse content\n- Modify content semantics\n- Couple scraping logic to a specific parser", + "docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes)\n from a source. It must not interpret or parse it.\n - A scraper is a stateless acquisition component that retrieves raw\n content from a source and returns it as a `Content` object.\n - Scrapers define how content is obtained, not what the content means.\n - Implementations may vary in transport mechanism, authentication\n strategy, retry and backoff behavior.\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics,\n or couple scraping logic to a specific parser.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.scraper.BaseScraper.fetch", "signature": "", - "docstring": "Fetch raw content from the given source.\n\nImplementations must retrieve the content referenced by `source`\nand return it as raw bytes wrapped in a `Content` object.\n\nArgs:\n source: Location identifier (URL, file path, S3 URI, etc.)\n metadata: Optional hints for the scraper (headers, auth, etc.)\n\nReturns:\n Content object containing raw bytes and metadata.\n - Raw content bytes\n - Source identifier\n - Optional metadata\n\nRaises:\n Exception: Retrieval-specific errors as defined by the implementation." + "docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.).\n\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.).\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source`\n and return it as raw bytes wrapped in a `Content` object." } } }, @@ -120,14 +120,14 @@ "kind": "class", "path": "omniread.pdf.scraper.BasePDFClient", "signature": "", - "docstring": "Abstract client responsible for retrieving PDF bytes\nfrom a specific backing store (filesystem, S3, FTP, etc.).\n\nImplementations must:\n- Accept a source identifier appropriate to the backing store\n- Return the full PDF binary payload\n- Raise retrieval-specific errors on failure", + "docstring": "Abstract client responsible for retrieving PDF bytes.\n\nRetrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).\n\nNotes:\n **Responsibilities:**\n\n - Implementations must accept a source identifier appropriate to\n the backing store.\n - Return the full PDF binary payload.\n - Raise retrieval-specific errors on failure.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.scraper.BasePDFClient.fetch", "signature": "", - "docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source: Identifier of the PDF location, such as a file path,\n object storage key, or remote reference.\n\nReturns:\n Raw PDF bytes.\n\nRaises:\n Exception: Retrieval-specific errors defined by the implementation." + "docstring": "Fetch raw PDF bytes from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF location, such as a file path, object storage key, or remote reference.\n\nReturns:\n bytes:\n Raw PDF bytes.\n\nRaises:\n Exception:\n Retrieval-specific errors defined by the implementation." } } }, @@ -135,15 +135,15 @@ "name": "PDFScraper", "kind": "class", "path": "omniread.pdf.scraper.PDFScraper", - "signature": "", - "docstring": "Scraper for PDF sources.\n\nDelegates byte retrieval to a PDF client and normalizes\noutput into Content.\n\nThe scraper:\n- Does not perform parsing or interpretation\n- Does not assume a specific storage backend\n- Preserves caller-provided metadata", + "signature": "", + "docstring": "Scraper for PDF sources.\n\nNotes:\n **Responsibilities:**\n\n - Delegates byte retrieval to a PDF client and normalizes output\n into `Content`.\n - Preserves caller-provided metadata.\n\n **Constraints:**\n\n - The scraper does not perform parsing or interpretation.\n - Does not assume a specific storage backend.", "members": { "fetch": { "name": "fetch", "kind": "function", "path": "omniread.pdf.scraper.PDFScraper.fetch", - "signature": "", - "docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source: Identifier of the PDF source as understood by the\n configured PDF client.\n metadata: Optional metadata to attach to the returned content.\n\nReturns:\n A `Content` instance containing:\n - Raw PDF bytes\n - Source identifier\n - PDF content type\n - Optional metadata\n\nRaises:\n Exception: Retrieval-specific errors raised by the PDF client." + "signature": "", + "docstring": "Fetch a PDF document from the given source.\n\nArgs:\n source (Any):\n Identifier of the PDF source as understood by the configured PDF client.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to attach to the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors raised by the PDF client." } } } diff --git a/mkdocs.yml b/mkdocs.yml index babd582..4ab42e7 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -8,12 +8,19 @@ theme: text: Inter code: JetBrains Mono features: - - navigation.tabs + - navigation.sections - navigation.expand - navigation.top - navigation.instant + - navigation.tracking + - navigation.indexes - content.code.copy - content.code.annotate + - content.tabs.link + - content.action.edit + - search.highlight + - search.share + - search.suggest plugins: - search - mkdocstrings: @@ -31,6 +38,30 @@ plugins: annotations_path: brief show_root_heading: true group_by_category: true + show_category_heading: true + show_object_full_path: false + show_symbol_type_heading: true +markdown_extensions: +- pymdownx.superfences +- pymdownx.inlinehilite +- pymdownx.snippets +- admonition +- pymdownx.details +- pymdownx.superfences +- pymdownx.highlight: + linenums: true + anchor_linenums: true + line_spans: __span + pygments_lang_class: true +- pymdownx.tabbed: + alternate_style: true +- pymdownx.tasklist: + custom_checkbox: true +- tables +- footnotes +- pymdownx.caret +- pymdownx.tilde +- pymdownx.mark site_name: omniread nav: - Home: index.md diff --git a/omniread/__init__.py b/omniread/__init__.py index 8108e09..46f53da 100644 --- a/omniread/__init__.py +++ b/omniread/__init__.py @@ -1,125 +1,132 @@ """ -OmniRead — format-agnostic content acquisition and parsing framework. +# Summary -OmniRead provides a **cleanly layered architecture** for fetching, parsing, +`OmniRead` — format-agnostic content acquisition and parsing framework. + +`OmniRead` provides a **cleanly layered architecture** for fetching, parsing, and normalizing content from heterogeneous sources such as HTML documents and PDF files. The library is structured around three core concepts: -1. **Content** - A canonical, format-agnostic container representing raw content bytes - and minimal contextual metadata. +1. **`Content`**: A canonical, format-agnostic container representing raw content + bytes and minimal contextual metadata. +2. **`Scrapers`**: Components responsible for *acquiring* raw content from a + source (HTTP, filesystem, object storage, etc.). `Scrapers` never interpret + content. +3. **`Parsers`**: Components responsible for *interpreting* acquired content and + converting it into structured, typed representations. -2. **Scrapers** - Components responsible for *acquiring* raw content from a source - (HTTP, filesystem, object storage, etc.). Scrapers never interpret - content. +`OmniRead` deliberately separates these responsibilities to ensure: -3. **Parsers** - Components responsible for *interpreting* acquired content and - converting it into structured, typed representations. +- Clear boundaries between IO and interpretation. +- Replaceable implementations per format. +- Predictable, testable behavior. -OmniRead deliberately separates these responsibilities to ensure: -- Clear boundaries between IO and interpretation -- Replaceable implementations per format -- Predictable, testable behavior +# Installation ----------------------------------------------------------------------- -Installation ----------------------------------------------------------------------- +Install `OmniRead` using pip: -Install OmniRead using pip: +```bash +pip install omniread +``` - pip install omniread +Install OmniRead using Poetry: +```bash +poetry add omniread +``` -Or with Poetry: +--- - poetry add omniread +## Quick start ----------------------------------------------------------------------- -Basic Usage ----------------------------------------------------------------------- +Example: + HTML example: + ```python + from omniread import HTMLScraper, HTMLParser -HTML example: + scraper = HTMLScraper() + content = scraper.fetch("https://example.com") - from omniread import HTMLScraper, HTMLParser + class TitleParser(HTMLParser[str]): + def parse(self) -> str: + return self._soup.title.string - scraper = HTMLScraper() - content = scraper.fetch("https://example.com") + parser = TitleParser(content) + title = parser.parse() + ``` - class TitleParser(HTMLParser[str]): - def parse(self) -> str: - return self._soup.title.string + PDF example: + ```python + from omniread import FileSystemPDFClient, PDFScraper, PDFParser + from pathlib import Path - parser = TitleParser(content) - title = parser.parse() + client = FileSystemPDFClient() + scraper = PDFScraper(client=client) + content = scraper.fetch(Path("document.pdf")) -PDF example: + class TextPDFParser(PDFParser[str]): + def parse(self) -> str: + # implement PDF text extraction + ... - from omniread import FileSystemPDFClient, PDFScraper, PDFParser - from pathlib import Path + parser = TextPDFParser(content) + result = parser.parse() + ``` - client = FileSystemPDFClient() - scraper = PDFScraper(client=client) - content = scraper.fetch(Path("document.pdf")) +--- - class TextPDFParser(PDFParser[str]): - def parse(self) -> str: - # implement PDF text extraction - ... - - parser = TextPDFParser(content) - result = parser.parse() - ----------------------------------------------------------------------- -Public API Surface ----------------------------------------------------------------------- +# Public API This module re-exports the **recommended public entry points** of OmniRead. - Consumers are encouraged to import from this namespace rather than from format-specific submodules directly, unless advanced customization is required. -Core: -- Content -- ContentType +- `Content`: Canonical content model. +- `ContentType`: Supported media types. +- `HTMLScraper`: HTTP-based HTML acquisition. +- `HTMLParser`: Base parser for HTML DOM interpretation. +- `FileSystemPDFClient`: Local filesystem PDF access. +- `PDFScraper`: PDF-specific content acquisition. +- `PDFParser`: Base parser for PDF binary interpretation. +- `FileSystemXlsxClient`: Local filesystem spreadsheet access. +- `XlsxScraper`: XLSX-specific content acquisition. +- `XlsxParser`: Generic string-row parser for xlsx workbooks. -HTML: -- HTMLScraper -- HTMLParser +--- -PDF: -- FileSystemPDFClient -- PDFScraper -- PDFParser - -## Core Philosophy +# Core Philosophy `OmniRead` is designed as a **decoupled content engine**: -1. **Separation of Concerns**: Scrapers *fetch*, Parsers *interpret*. Neither knows about the other. -2. **Normalized Exchange**: All components communicate via the `Content` model, ensuring a consistent contract. -3. **Format Agnosticism**: The core logic is independent of whether the input is HTML, PDF, or JSON. +1. **Separation of Concerns**: Scrapers *fetch*, Parsers *interpret*. Neither + knows about the other. +2. **Normalized Exchange**: All components communicate via the `Content` model, + ensuring a consistent contract. +3. **Format Agnosticism**: The core logic is independent of whether the input + is HTML, PDF, or JSON. -## Documentation Design - -For those extending `OmniRead`, follow these "AI-Native" docstring principles: - -### For Humans -- **Clear Contracts**: Explicitly state what a component is and is NOT responsible for. -- **Runnable Examples**: Include small, logical snippets in the package `__init__.py`. - -### For LLMs -- **Structured Models**: Use dataclasses and enums for core data to ensure clean MCP JSON representation. -- **Type Safety**: All public APIs must be fully typed and have corresponding `.pyi` stubs. -- **Detailed Raises**: Include `: description` pairs in the `Raises` section to help agents handle errors gracefully. +--- """ from .core import Content, ContentType +from .csv import ( + BaseCsvClient, + CsvParser, + CsvParserBase, + CsvScraper, + FileSystemCsvClient, +) from .html import HTMLScraper, HTMLParser from .pdf import FileSystemPDFClient, PDFScraper, PDFParser +from .xlsx import ( + BaseXlsxClient, + FileSystemXlsxClient, + XlsxParser, + XlsxParserBase, + XlsxScraper, +) __all__ = [ # core @@ -134,4 +141,18 @@ __all__ = [ "FileSystemPDFClient", "PDFScraper", "PDFParser", + + # csv + "BaseCsvClient", + "FileSystemCsvClient", + "CsvScraper", + "CsvParser", + "CsvParserBase", + + # xlsx + "BaseXlsxClient", + "FileSystemXlsxClient", + "XlsxScraper", + "XlsxParser", + "XlsxParserBase", ] diff --git a/omniread/core/__init__.py b/omniread/core/__init__.py index ea9f4b4..08a580c 100644 --- a/omniread/core/__init__.py +++ b/omniread/core/__init__.py @@ -1,4 +1,6 @@ """ +# Summary + Core domain contracts for OmniRead. This package defines the **format-agnostic domain layer** of OmniRead. @@ -9,11 +11,21 @@ Public exports from this package are considered **stable contracts** and are safe for downstream consumers to depend on. Submodules: -- content: Canonical content models and enums -- parser: Abstract parsing contracts -- scraper: Abstract scraping contracts + +- `content`: Canonical content models and enums. +- `parser`: Abstract parsing contracts. +- `scraper`: Abstract scraping contracts. Format-specific behavior must not be introduced at this layer. + +--- + +# Public API + +- `Content` +- `ContentType` + +--- """ from .content import Content, ContentType diff --git a/omniread/core/content.py b/omniread/core/content.py index 2bc1af1..775a843 100644 --- a/omniread/core/content.py +++ b/omniread/core/content.py @@ -1,4 +1,6 @@ """ +# Summary + Canonical content models for OmniRead. This module defines the **format-agnostic content representation** used across @@ -18,9 +20,13 @@ class ContentType(str, Enum): """ Supported MIME types for extracted content. - This enum represents the declared or inferred media type of the content - source. It is primarily used for routing content to the appropriate - parser or downstream consumer. + Notes: + **Guarantees:** + + - This enum represents the declared or inferred media type of the + content source. + - It is primarily used for routing content to the appropriate + parser or downstream consumer. """ HTML = "text/html" @@ -29,6 +35,12 @@ class ContentType(str, Enum): PDF = "application/pdf" """PDF document content.""" + XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + """Office Open XML spreadsheet (xlsx/xlsm) content.""" + + CSV = "text/csv" + """Comma-separated-value document content.""" + JSON = "application/json" """JSON document content.""" @@ -41,23 +53,31 @@ class Content: """ Normalized representation of extracted content. - A `Content` instance represents a raw content payload along with minimal - contextual metadata describing its origin and type. + Notes: + **Responsibilities:** - This class is the **primary exchange format** between: - - Scrapers - - Parsers - - Downstream consumers - - Attributes: - raw: Raw content bytes as retrieved from the source. - source: Identifier of the content origin (URL, file path, or logical name). - content_type: Optional MIME type of the content, if known. - metadata: Optional, implementation-defined metadata associated with - the content (e.g., headers, encoding hints, extraction notes). + - A `Content` instance represents a raw content payload along with + minimal contextual metadata describing its origin and type. + - This class is the primary exchange format between scrapers, + parsers, and downstream consumers. """ raw: bytes + """ + Raw content bytes as retrieved from the source. + """ + source: str + """ + Identifier of the content origin (URL, file path, or logical name). + """ + content_type: Optional[ContentType] = None + """ + Optional MIME type of the content, if known. + """ + metadata: Optional[Mapping[str, Any]] = None + """ + Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes). + """ diff --git a/omniread/core/parser.py b/omniread/core/parser.py index 4f0bc08..7c32993 100644 --- a/omniread/core/parser.py +++ b/omniread/core/parser.py @@ -1,15 +1,19 @@ """ +# Summary + Abstract parsing contracts for OmniRead. This module defines the **format-agnostic parser interface** used to transform raw content into structured, typed representations. Parsers are responsible for: + - Interpreting a single `Content` instance - Validating compatibility with the content type - Producing a structured output suitable for downstream consumers Parsers are not responsible for: + - Fetching or acquiring content - Performing retries or error recovery - Managing multiple content sources @@ -27,23 +31,24 @@ class BaseParser(ABC, Generic[T]): """ Base interface for all parsers. - A parser is a self-contained object that owns the Content - it is responsible for interpreting. + Notes: + **Guarantees:** - Implementations must: - - Declare supported content types via `supported_types` - - Raise parsing-specific exceptions from `parse()` - - Remain deterministic for a given input + - A parser is a self-contained object that owns the `Content` it is + responsible for interpreting. + - Consumers may rely on early validation of content compatibility + and type-stable return values from `parse()`. - Consumers may rely on: - - Early validation of content compatibility - - Type-stable return values from `parse()` + **Responsibilities:** + + - Implementations must declare supported content types via `supported_types`. + - Implementations must raise parsing-specific exceptions from `parse()`. + - Implementations must remain deterministic for a given input. """ supported_types: Set[ContentType] = set() - """Set of content types supported by this parser. - - An empty set indicates that the parser is content-type agnostic. + """ + Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic. """ def __init__(self, content: Content): @@ -51,10 +56,12 @@ class BaseParser(ABC, Generic[T]): Initialize the parser with content to be parsed. Args: - content: Content instance to be parsed. + content (Content): + Content instance to be parsed. Raises: - ValueError: If the content type is not supported by this parser. + ValueError: + If the content type is not supported by this parser. """ self.content = content @@ -70,14 +77,19 @@ class BaseParser(ABC, Generic[T]): """ Parse the owned content into structured output. - Implementations must fully consume the provided content and - return a deterministic, structured output. - Returns: - Parsed, structured representation. + T: + Parsed, structured representation. Raises: - Exception: Parsing-specific errors as defined by the implementation. + Exception: + Parsing-specific errors as defined by the implementation. + + Notes: + **Responsibilities:** + + - Implementations must fully consume the provided content and + return a deterministic, structured output. """ raise NotImplementedError @@ -86,7 +98,8 @@ class BaseParser(ABC, Generic[T]): Check whether this parser supports the content's type. Returns: - True if the content type is supported; False otherwise. + bool: + True if the content type is supported; False otherwise. """ if not self.supported_types: diff --git a/omniread/core/scraper.py b/omniread/core/scraper.py index 910dfe2..fd4bd81 100644 --- a/omniread/core/scraper.py +++ b/omniread/core/scraper.py @@ -1,15 +1,19 @@ """ +# Summary + Abstract scraping contracts for OmniRead. This module defines the **format-agnostic scraper interface** responsible for acquiring raw content from external sources. Scrapers are responsible for: + - Locating and retrieving raw content bytes - Attaching minimal contextual metadata - Returning normalized `Content` objects Scrapers are explicitly NOT responsible for: + - Parsing or interpreting content - Inferring structure or semantics - Performing content-type specific processing @@ -27,23 +31,21 @@ class BaseScraper(ABC): """ Base interface for all scrapers. - A scraper is responsible ONLY for fetching raw content - (bytes) from a source. It must not interpret or parse it. + Notes: + **Responsibilities:** - A scraper is a **stateless acquisition component** that retrieves raw - content from a source and returns it as a `Content` object. + - A scraper is responsible ONLY for fetching raw content (bytes) + from a source. It must not interpret or parse it. + - A scraper is a stateless acquisition component that retrieves raw + content from a source and returns it as a `Content` object. + - Scrapers define how content is obtained, not what the content means. + - Implementations may vary in transport mechanism, authentication + strategy, retry and backoff behavior. - Scrapers define *how content is obtained*, not *what the content means*. + **Constraints:** - Implementations may vary in: - - Transport mechanism (HTTP, filesystem, cloud storage) - - Authentication strategy - - Retry and backoff behavior - - Implementations must not: - - Parse content - - Modify content semantics - - Couple scraping logic to a specific parser + - Implementations must not parse content, modify content semantics, + or couple scraping logic to a specific parser. """ @abstractmethod @@ -56,20 +58,25 @@ class BaseScraper(ABC): """ Fetch raw content from the given source. - Implementations must retrieve the content referenced by `source` - and return it as raw bytes wrapped in a `Content` object. - Args: - source: Location identifier (URL, file path, S3 URI, etc.) - metadata: Optional hints for the scraper (headers, auth, etc.) + source (str): + Location identifier (URL, file path, S3 URI, etc.). + + metadata (Optional[Mapping[str, Any]], optional): + Optional hints for the scraper (headers, auth, etc.). Returns: - Content object containing raw bytes and metadata. - - Raw content bytes - - Source identifier - - Optional metadata + Content: + Content object containing raw bytes and metadata. Raises: - Exception: Retrieval-specific errors as defined by the implementation. + Exception: + Retrieval-specific errors as defined by the implementation. + + Notes: + **Responsibilities:** + + - Implementations must retrieve the content referenced by `source` + and return it as raw bytes wrapped in a `Content` object. """ raise NotImplementedError diff --git a/omniread/csv/__init__.py b/omniread/csv/__init__.py new file mode 100644 index 0000000..ae704d2 --- /dev/null +++ b/omniread/csv/__init__.py @@ -0,0 +1,26 @@ +""" +# Summary + +CSV subpackage for OmniRead. + +Provides acquisition and parsing of comma-separated-value content: + +- `BaseCsvClient`: abstract backing-store client for csv bytes. +- `FileSystemCsvClient`: local filesystem implementation. +- `CsvScraper`: wraps fetched bytes into canonical `Content`. +- `CsvParserBase`: content-type-enforcing parser contract. +- `CsvParser`: generic string-row parser built on the standard csv module. +""" + +from .client import BaseCsvClient, FileSystemCsvClient +from .parser import CsvParser +from .parser_base import CsvParserBase +from .scraper import CsvScraper + +__all__ = [ + "BaseCsvClient", + "FileSystemCsvClient", + "CsvScraper", + "CsvParser", + "CsvParserBase", +] diff --git a/omniread/csv/client.py b/omniread/csv/client.py new file mode 100644 index 0000000..22ad75f --- /dev/null +++ b/omniread/csv/client.py @@ -0,0 +1,97 @@ +""" +# Summary + +CSV client abstractions for OmniRead. + +This module defines the **client layer** responsible for retrieving raw +comma-separated-value document bytes from a concrete backing store. + +Clients provide low-level access to csv binaries and are intentionally +decoupled from scraping and parsing logic. They do not perform validation, +interpretation, or content extraction. + +Typical backing stores include: + +- Local filesystems +- Object storage (S3, GCS, etc.) +- Network file systems +""" + +from typing import Any +from abc import ABC, abstractmethod +from pathlib import Path + + +class BaseCsvClient(ABC): + """ + Abstract client responsible for retrieving csv bytes. + + Retrieves bytes from a specific backing store (filesystem, S3, FTP, etc.). + + Notes: + **Responsibilities:** + + - Implementations must accept a source identifier appropriate to + the backing store. + - Return the full csv binary payload. + - Raise retrieval-specific errors on failure. + """ + + @abstractmethod + def fetch(self, source: Any) -> bytes: + """ + Fetch raw csv bytes from the given source. + + Args: + source (Any): + Identifier of the csv location, such as a file path, + object storage key, or remote reference. + + Returns: + bytes: + Raw csv bytes. + + Raises: + Exception: + Retrieval-specific errors defined by the implementation. + """ + raise NotImplementedError + + +class FileSystemCsvClient(BaseCsvClient): + """ + CSV client that reads from the local filesystem. + + Notes: + **Guarantees:** + + - This client reads csv files directly from the disk and + returns their raw binary contents. + """ + + def fetch(self, path: Path) -> bytes: + """ + Read a csv file from the local filesystem. + + Args: + path (Path): + Filesystem path to the csv file. + + Returns: + bytes: + Raw csv bytes. + + Raises: + FileNotFoundError: + If the path does not exist. + ValueError: + If the path exists but is not a file. + """ + + if not path.exists(): + raise FileNotFoundError(f"csv not found: {path}") + + if not path.is_file(): + raise ValueError(f"Path is not a file: {path}") + + return path.read_bytes() diff --git a/omniread/csv/parser.py b/omniread/csv/parser.py new file mode 100644 index 0000000..358e664 --- /dev/null +++ b/omniread/csv/parser.py @@ -0,0 +1,102 @@ +""" +# Summary + +CSV parser implementations for OmniRead. + +This module provides a concrete, generic parser for comma-separated-value +documents. It exposes records as lists of string cells so downstream +consumers can interpret tabular content without depending on the ``csv`` +module directly. + +The parser is intentionally statement-agnostic: it performs no header +detection or column interpretation beyond basic cell normalization and +delimiter detection. +""" + +from io import StringIO +from csv import Sniffer, reader +from typing import List + +from omniread.core.content import Content +from .parser_base import CsvParserBase + + +class CsvParser(CsvParserBase): + """ + Generic csv parser producing string rows from the document. + + Notes: + **Responsibilities:** + + - Decode the payload (UTF-8 with BOM support, Latin-1 fallback). + - Detect the delimiter from a leading sample (`,` `;` tab `|`), + defaulting to `,`. + - Normalize cells into deterministic stripped string values. + - Expose row extraction helpers mirroring `XlsxParser.rows`. + + **Constraints:** + + - All values are strings; consumers requiring typed values must + convert on their side. + - Quoted fields containing delimiters/newlines are handled by + the standard ``csv`` module. + """ + + _DELIMITERS = ",;\t|" + + def __init__(self, content: Content): + """ + Initialize the parser. + + Args: + content (Content): + CSV content to parse; its type must be supported. + """ + super().__init__(content) + + def parse(self) -> List[List[str]]: + """ + Parse the document into normalized string rows. + + Returns: + List[List[str]]: + Rows of the document. + """ + return self.rows() + + def rows(self, *, skip_empty: bool = True) -> List[List[str]]: + """ + Extract normalized string rows from the document. + + Args: + skip_empty (bool): + When True (default), rows whose cells are all blank are + omitted. + + Returns: + List[List[str]]: + Normalized rows; trailing blank cells are trimmed per row. + """ + text = self._decode(self.content.raw) + dialect_sample = text[:4096] + try: + dialect = Sniffer().sniff(dialect_sample, delimiters=self._DELIMITERS) + delimiter = dialect.delimiter + except Exception: + delimiter = "," + out: List[List[str]] = [] + for row in reader(StringIO(text), delimiter=delimiter): + cells = [c.strip() for c in row] + while cells and not cells[-1]: + cells.pop() + if skip_empty and not any(cells): + continue + out.append(cells) + return out + + @staticmethod + def _decode(raw: bytes) -> str: + try: + return raw.decode("utf-8-sig") + except UnicodeDecodeError: + return raw.decode("latin-1") diff --git a/omniread/csv/parser_base.py b/omniread/csv/parser_base.py new file mode 100644 index 0000000..5daf885 --- /dev/null +++ b/omniread/csv/parser_base.py @@ -0,0 +1,55 @@ +""" +# Summary + +CSV parser base implementation for OmniRead. + +This module defines the **CSV-specific parser contract**, extending the +format-agnostic `BaseParser` with constraints appropriate for +comma-separated-value documents. +""" + +from typing import Generic, TypeVar +from abc import abstractmethod + +from omniread.core.content import ContentType +from omniread.core.parser import BaseParser + +T = TypeVar("T") + + +class CsvParserBase(BaseParser[T], Generic[T]): + """ + Base csv parser. + + Notes: + **Responsibilities:** + + - This class enforces csv content-type compatibility and provides + the extension point for implementing concrete csv parsing + strategies. + + **Constraints:** + + - Concrete implementations must define the output type `T` and + implement the `parse()` method. + """ + + supported_types = {ContentType.CSV} + """ + Set of content types supported by this parser (CSV only). + """ + + @abstractmethod + def parse(self) -> T: + """ + Parse csv content into a structured output. + + Returns: + T: + Parsed representation of type `T`. + + Raises: + Exception: + Parsing-specific errors as defined by the implementation. + """ + raise NotImplementedError diff --git a/omniread/csv/scraper.py b/omniread/csv/scraper.py new file mode 100644 index 0000000..5785dfc --- /dev/null +++ b/omniread/csv/scraper.py @@ -0,0 +1,79 @@ +""" +# Summary + +CSV scraper for OmniRead. + +This module defines the scraper responsible for acquiring raw +comma-separated-value document content from a backing store via a +configured client. + +The scraper does not interpret or parse the acquired bytes; it wraps them in +the canonical `Content` model. +""" + +from typing import Any, Mapping, Optional + +from omniread.core.content import Content, ContentType +from .client import BaseCsvClient + + +class CsvScraper: + """ + Scraper for csv documents. + + Notes: + **Responsibilities:** + + - Fetch raw csv bytes via the configured client. + - Wrap the payload in a canonical `Content` instance with the + CSV content type and source identifier. + + **Constraints:** + + - The scraper does not perform parsing or interpretation. + - Does not assume a specific storage backend. + """ + + def __init__(self, *, client: BaseCsvClient): + """ + Initialize the CSV scraper. + + Args: + client (BaseCsvClient): + Client responsible for retrieving raw csv bytes. + """ + self._client = client + + def fetch( + self, + source: Any, + *, + metadata: Optional[Mapping[str, Any]] = None, + ) -> Content: + """ + Fetch a csv document from the given source. + + Args: + source (Any): + Identifier of the csv source as understood by the + configured client. + metadata (Optional[Mapping[str, Any]], optional): + Optional metadata to attach to the returned content. + + Returns: + Content: + A `Content` instance containing raw csv bytes, source + identifier, CSV content type, and optional metadata. + + Raises: + Exception: + Retrieval-specific errors raised by the client. + """ + raw = self._client.fetch(source) + + return Content( + raw=raw, + source=source, + content_type=ContentType.CSV, + metadata=dict(metadata) if metadata else None, + ) diff --git a/omniread/html/__init__.py b/omniread/html/__init__.py index 4ef38b9..bd9dae0 100644 --- a/omniread/html/__init__.py +++ b/omniread/html/__init__.py @@ -1,20 +1,33 @@ """ +# Summary + HTML format implementation for OmniRead. This package provides **HTML-specific implementations** of the core OmniRead contracts defined in `omniread.core`. It includes: -- HTML parsers that interpret HTML content -- HTML scrapers that retrieve HTML documents -This package: -- Implements, but does not redefine, core contracts -- May contain HTML-specific behavior and edge-case handling -- Produces canonical content models defined in `omniread.core.content` +- HTML parsers that interpret HTML content. +- HTML scrapers that retrieve HTML documents. + +Key characteristics: + +- Implements, but does not redefine, core contracts. +- May contain HTML-specific behavior and edge-case handling. +- Produces canonical content models defined in `omniread.core.content`. Consumers should depend on `omniread.core` interfaces wherever possible and use this package only when HTML-specific behavior is required. + +--- + +# Public API + +- `HTMLScraper` +- `HTMLParser` + +--- """ diff --git a/omniread/html/parser.py b/omniread/html/parser.py index 06e25e6..de9b49b 100644 --- a/omniread/html/parser.py +++ b/omniread/html/parser.py @@ -1,10 +1,13 @@ """ +# Summary + HTML parser base implementations for OmniRead. This module provides reusable HTML parsing utilities built on top of the abstract parser contracts defined in `omniread.core.parser`. It supplies: + - Content-type enforcement for HTML inputs - BeautifulSoup initialization and lifecycle management - Common helper methods for extracting structured data from HTML elements @@ -28,36 +31,44 @@ class HTMLParser(BaseParser[T], Generic[T]): """ Base HTML parser. - This class extends the core `BaseParser` with HTML-specific behavior, - including DOM parsing via BeautifulSoup and reusable extraction helpers. + Notes: + **Responsibilities:** - Provides reusable helpers for HTML extraction. - Concrete parsers must explicitly define the return type. + - This class extends the core `BaseParser` with HTML-specific behavior, + including DOM parsing via BeautifulSoup and reusable extraction helpers. + - Provides reusable helpers for HTML extraction. Concrete parsers must + explicitly define the return type. - Characteristics: - - Accepts only HTML content - - Owns a parsed BeautifulSoup DOM tree - - Provides pure helper utilities for common HTML structures + **Guarantees:** - Concrete subclasses must: - - Define the output type `T` - - Implement the `parse()` method + - Accepts only HTML content. + - Owns a parsed BeautifulSoup DOM tree. + - Provides pure helper utilities for common HTML structures. + + **Constraints:** + + - Concrete subclasses must define the output type `T` and implement + the `parse()` method. """ supported_types = {ContentType.HTML} - """Set of content types supported by this parser (HTML only).""" + """ + Set of content types supported by this parser (HTML only). + """ def __init__(self, content: Content, features: str = "html.parser"): """ Initialize the HTML parser. Args: - content: HTML content to be parsed. - features: BeautifulSoup parser backend to use - (e.g., 'html.parser', 'lxml'). + content (Content): + HTML content to be parsed. + features (str, optional): + BeautifulSoup parser backend to use (e.g., 'html.parser', 'lxml'). Raises: - ValueError: If the content is empty or not valid HTML. + ValueError: + If the content is empty or not valid HTML. """ super().__init__(content) self._features = features @@ -72,11 +83,15 @@ class HTMLParser(BaseParser[T], Generic[T]): """ Fully parse the HTML content into structured output. - Implementations must fully interpret the HTML DOM and return - a deterministic, structured output. - Returns: - Parsed representation of type `T`. + T: + Parsed representation of type `T`. + + Notes: + **Responsibilities:** + + - Implementations must fully interpret the HTML DOM and return a + deterministic, structured output. """ raise NotImplementedError @@ -90,11 +105,14 @@ class HTMLParser(BaseParser[T], Generic[T]): Extract normalized text from a `
` element. Args: - div: BeautifulSoup tag representing a `
`. - separator: String used to separate text nodes. + div (Tag): + BeautifulSoup tag representing a `
`. + table (Tag): + BeautifulSoup tag representing a `
`. Returns: - A list of rows, where each row is a list of cell text values. + list[list[str]]: + A list of rows, where each row is a list of cell text values. """ rows: list[list[str]] = [] for tr in table.find_all("tr"): @@ -141,10 +163,12 @@ class HTMLParser(BaseParser[T], Generic[T]): Build a BeautifulSoup DOM tree from raw HTML content. Returns: - Parsed BeautifulSoup document tree. + BeautifulSoup: + Parsed BeautifulSoup document tree. Raises: - ValueError: If the content payload is empty. + ValueError: + If the content payload is empty. """ if not self.content.raw: raise ValueError("Empty HTML content") @@ -154,12 +178,16 @@ class HTMLParser(BaseParser[T], Generic[T]): """ Extract high-level metadata from the HTML document. - This includes: - - Document title - - `` tag name/property → content mappings - Returns: - Dictionary containing extracted metadata. + dict[str, Any]: + Dictionary containing extracted metadata. + + Notes: + **Responsibilities:** + + - Extract high-level metadata from the HTML document. + - This includes: Document title, `` tag name/property to + content mappings. """ soup = self._soup diff --git a/omniread/html/scraper.py b/omniread/html/scraper.py index 58115b0..8d13635 100644 --- a/omniread/html/scraper.py +++ b/omniread/html/scraper.py @@ -1,4 +1,6 @@ """ +# Summary + HTML scraping implementation for OmniRead. This module provides an HTTP-based scraper for retrieving HTML documents. @@ -6,11 +8,13 @@ It implements the core `BaseScraper` contract using `httpx` as the transport layer. This scraper is responsible for: + - Fetching raw HTML bytes over HTTP(S) - Validating response content type - Attaching HTTP metadata to the returned content This scraper is not responsible for: + - Parsing or interpreting HTML - Retrying failed requests - Managing crawl policies or rate limiting @@ -25,21 +29,21 @@ from omniread.core.scraper import BaseScraper class HTMLScraper(BaseScraper): """ - Base HTML scraper using httpx. + Base HTML scraper using `httpx`. - This scraper retrieves HTML documents over HTTP(S) and returns them - as raw content wrapped in a `Content` object. + Notes: + **Responsibilities:** - Fetches raw bytes and metadata only. - The scraper: - - Uses `httpx.Client` for HTTP requests - - Enforces an HTML content type - - Preserves HTTP response metadata + - This scraper retrieves HTML documents over HTTP(S) and returns + them as raw content wrapped in a `Content` object. + - Fetches raw bytes and metadata only. + - The scraper uses `httpx.Client` for HTTP requests, enforces an + HTML content type, and preserves HTTP response metadata. - The scraper does not: - - Parse HTML - - Perform retries or backoff - - Handle non-HTML responses + **Constraints:** + + - The scraper does not: Parse HTML, perform retries or backoff, + handle non-HTML responses. """ def __init__( @@ -54,11 +58,14 @@ class HTMLScraper(BaseScraper): Initialize the HTML scraper. Args: - client: Optional pre-configured `httpx.Client`. If omitted, - a client is created internally. - timeout: Request timeout in seconds. - headers: Optional default HTTP headers. - follow_redirects: Whether to follow HTTP redirects. + client (httpx.Client | None, optional): + Optional pre-configured `httpx.Client`. If omitted, a client is created internally. + timeout (float, optional): + Request timeout in seconds. + headers (Optional[Mapping[str, str]], optional): + Optional default HTTP headers. + follow_redirects (bool, optional): + Whether to follow HTTP redirects. """ self._client = client or httpx.Client( @@ -76,11 +83,12 @@ class HTMLScraper(BaseScraper): Validate that the HTTP response contains HTML content. Args: - response: HTTP response returned by `httpx`. + response (httpx.Response): + HTTP response returned by `httpx`. Raises: - ValueError: If the `Content-Type` header is missing or does not - indicate HTML content. + ValueError: + If the `Content-Type` header is missing or does not indicate HTML content. """ raw_ct = response.headers.get("Content-Type") @@ -103,19 +111,20 @@ class HTMLScraper(BaseScraper): Fetch an HTML document from the given source. Args: - source: URL of the HTML document. - metadata: Optional metadata to be merged into the returned content. + source (str): + URL of the HTML document. + metadata (Optional[Mapping[str, Any]], optional): + Optional metadata to be merged into the returned content. Returns: - A `Content` instance containing: - - Raw HTML bytes - - Source URL - - HTML content type - - HTTP response metadata + Content: + A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata. Raises: - httpx.HTTPError: If the HTTP request fails. - ValueError: If the response is not valid HTML. + httpx.HTTPError: + If the HTTP request fails. + ValueError: + If the response is not valid HTML. """ response = self._client.get(source) diff --git a/omniread/pdf/__init__.py b/omniread/pdf/__init__.py index d924554..fd22c31 100644 --- a/omniread/pdf/__init__.py +++ b/omniread/pdf/__init__.py @@ -1,4 +1,6 @@ """ +# Summary + PDF format implementation for OmniRead. This package provides **PDF-specific implementations** of the core OmniRead @@ -6,12 +8,23 @@ contracts defined in `omniread.core`. Unlike HTML, PDF handling requires an explicit client layer for document access. This package therefore includes: -- PDF clients for acquiring raw PDF data -- PDF scrapers that coordinate client access -- PDF parsers that extract structured content from PDF binaries + +- PDF clients for acquiring raw PDF data. +- PDF scrapers that coordinate client access. +- PDF parsers that extract structured content from PDF binaries. Public exports from this package represent the supported PDF pipeline and are safe for consumers to import directly when working with PDFs. + +--- + +# Public API + +- `FileSystemPDFClient` +- `PDFScraper` +- `PDFParser` + +--- """ from .client import FileSystemPDFClient diff --git a/omniread/pdf/client.py b/omniread/pdf/client.py index c1de901..ee93c64 100644 --- a/omniread/pdf/client.py +++ b/omniread/pdf/client.py @@ -1,4 +1,6 @@ """ +# Summary + PDF client abstractions for OmniRead. This module defines the **client layer** responsible for retrieving raw PDF @@ -9,6 +11,7 @@ decoupled from scraping and parsing logic. They do not perform validation, interpretation, or content extraction. Typical backing stores include: + - Local filesystems - Object storage (S3, GCS, etc.) - Network file systems @@ -21,13 +24,17 @@ from pathlib import Path class BasePDFClient(ABC): """ - Abstract client responsible for retrieving PDF bytes - from a specific backing store (filesystem, S3, FTP, etc.). + Abstract client responsible for retrieving PDF bytes. - Implementations must: - - Accept a source identifier appropriate to the backing store - - Return the full PDF binary payload - - Raise retrieval-specific errors on failure + Retrieves bytes from a specific backing store (filesystem, S3, FTP, etc.). + + Notes: + **Responsibilities:** + + - Implementations must accept a source identifier appropriate to + the backing store. + - Return the full PDF binary payload. + - Raise retrieval-specific errors on failure. """ @abstractmethod @@ -36,14 +43,16 @@ class BasePDFClient(ABC): Fetch raw PDF bytes from the given source. Args: - source: Identifier of the PDF location, such as a file path, - object storage key, or remote reference. + source (Any): + Identifier of the PDF location, such as a file path, object storage key, or remote reference. Returns: - Raw PDF bytes. + bytes: + Raw PDF bytes. Raises: - Exception: Retrieval-specific errors defined by the implementation. + Exception: + Retrieval-specific errors defined by the implementation. """ raise NotImplementedError @@ -52,8 +61,11 @@ class FileSystemPDFClient(BasePDFClient): """ PDF client that reads from the local filesystem. - This client reads PDF files directly from the disk and returns their raw - binary contents. + Notes: + **Guarantees:** + + - This client reads PDF files directly from the disk and returns + their raw binary contents. """ def fetch(self, path: Path) -> bytes: @@ -61,14 +73,18 @@ class FileSystemPDFClient(BasePDFClient): Read a PDF file from the local filesystem. Args: - path: Filesystem path to the PDF file. + path (Path): + Filesystem path to the PDF file. Returns: - Raw PDF bytes. + bytes: + Raw PDF bytes. Raises: - FileNotFoundError: If the path does not exist. - ValueError: If the path exists but is not a file. + FileNotFoundError: + If the path does not exist. + ValueError: + If the path exists but is not a file. """ if not path.exists(): diff --git a/omniread/pdf/parser.py b/omniread/pdf/parser.py index 65c4b5a..2e010a8 100644 --- a/omniread/pdf/parser.py +++ b/omniread/pdf/parser.py @@ -1,4 +1,6 @@ """ +# Summary + PDF parser base implementations for OmniRead. This module defines the **PDF-specific parser contract**, extending the @@ -21,29 +23,40 @@ class PDFParser(BaseParser[T], Generic[T]): """ Base PDF parser. - This class enforces PDF content-type compatibility and provides the - extension point for implementing concrete PDF parsing strategies. + Notes: + **Responsibilities:** - Concrete implementations must define: - - Define the output type `T` - - Implement the `parse()` method + - This class enforces PDF content-type compatibility and provides + the extension point for implementing concrete PDF parsing strategies. + + **Constraints:** + + - Concrete implementations must define the output type `T` and + implement the `parse()` method. """ supported_types = {ContentType.PDF} - """Set of content types supported by this parser (PDF only).""" + """ + Set of content types supported by this parser (PDF only). + """ @abstractmethod def parse(self) -> T: """ Parse PDF content into a structured output. - Implementations must fully interpret the PDF binary payload and - return a deterministic, structured output. - Returns: - Parsed representation of type `T`. + T: + Parsed representation of type `T`. Raises: - Exception: Parsing-specific errors as defined by the implementation. + Exception: + Parsing-specific errors as defined by the implementation. + + Notes: + **Responsibilities:** + + - Implementations must fully interpret the PDF binary payload and + return a deterministic, structured output. """ raise NotImplementedError diff --git a/omniread/pdf/scraper.py b/omniread/pdf/scraper.py index 7446ef0..4726580 100644 --- a/omniread/pdf/scraper.py +++ b/omniread/pdf/scraper.py @@ -1,4 +1,6 @@ """ +# Summary + PDF scraping implementation for OmniRead. This module provides a PDF-specific scraper that coordinates PDF byte @@ -19,13 +21,17 @@ class PDFScraper(BaseScraper): """ Scraper for PDF sources. - Delegates byte retrieval to a PDF client and normalizes - output into Content. + Notes: + **Responsibilities:** - The scraper: - - Does not perform parsing or interpretation - - Does not assume a specific storage backend - - Preserves caller-provided metadata + - Delegates byte retrieval to a PDF client and normalizes output + into `Content`. + - Preserves caller-provided metadata. + + **Constraints:** + + - The scraper does not perform parsing or interpretation. + - Does not assume a specific storage backend. """ def __init__(self, *, client: BasePDFClient): @@ -33,7 +39,8 @@ class PDFScraper(BaseScraper): Initialize the PDF scraper. Args: - client: PDF client responsible for retrieving raw PDF bytes. + client (BasePDFClient): + PDF client responsible for retrieving raw PDF bytes. """ self._client = client @@ -47,19 +54,18 @@ class PDFScraper(BaseScraper): Fetch a PDF document from the given source. Args: - source: Identifier of the PDF source as understood by the - configured PDF client. - metadata: Optional metadata to attach to the returned content. + source (Any): + Identifier of the PDF source as understood by the configured PDF client. + metadata (Optional[Mapping[str, Any]], optional): + Optional metadata to attach to the returned content. Returns: - A `Content` instance containing: - - Raw PDF bytes - - Source identifier - - PDF content type - - Optional metadata + Content: + A `Content` instance containing raw PDF bytes, source identifier, PDF content type, and optional metadata. Raises: - Exception: Retrieval-specific errors raised by the PDF client. + Exception: + Retrieval-specific errors raised by the PDF client. """ raw = self._client.fetch(source) diff --git a/omniread/xlsx/__init__.py b/omniread/xlsx/__init__.py new file mode 100644 index 0000000..a103c66 --- /dev/null +++ b/omniread/xlsx/__init__.py @@ -0,0 +1,27 @@ +""" +# Summary + +XLSX subpackage for OmniRead. + +Provides acquisition and parsing of Office Open XML spreadsheet (xlsx) +content: + +- `BaseXlsxClient`: abstract backing-store client for xlsx bytes. +- `FileSystemXlsxClient`: local filesystem implementation. +- `XlsxScraper`: wraps fetched bytes into canonical `Content`. +- `XlsxParserBase`: content-type-enforcing parser contract. +- `XlsxParser`: generic string-row parser built on openpyxl. +""" + +from .client import BaseXlsxClient, FileSystemXlsxClient +from .parser import XlsxParser +from .parser_base import XlsxParserBase +from .scraper import XlsxScraper + +__all__ = [ + "BaseXlsxClient", + "FileSystemXlsxClient", + "XlsxScraper", + "XlsxParser", + "XlsxParserBase", +] diff --git a/omniread/xlsx/client.py b/omniread/xlsx/client.py new file mode 100644 index 0000000..b7c9409 --- /dev/null +++ b/omniread/xlsx/client.py @@ -0,0 +1,97 @@ +""" +# Summary + +XLSX client abstractions for OmniRead. + +This module defines the **client layer** responsible for retrieving raw +Office Open XML spreadsheet bytes from a concrete backing store. + +Clients provide low-level access to xlsx binaries and are intentionally +decoupled from scraping and parsing logic. They do not perform validation, +interpretation, or content extraction. + +Typical backing stores include: + +- Local filesystems +- Object storage (S3, GCS, etc.) +- Network file systems +""" + +from typing import Any +from abc import ABC, abstractmethod +from pathlib import Path + + +class BaseXlsxClient(ABC): + """ + Abstract client responsible for retrieving spreadsheet bytes. + + Retrieves bytes from a specific backing store (filesystem, S3, FTP, etc.). + + Notes: + **Responsibilities:** + + - Implementations must accept a source identifier appropriate to + the backing store. + - Return the full xlsx binary payload. + - Raise retrieval-specific errors on failure. + """ + + @abstractmethod + def fetch(self, source: Any) -> bytes: + """ + Fetch raw xlsx bytes from the given source. + + Args: + source (Any): + Identifier of the spreadsheet location, such as a file path, + object storage key, or remote reference. + + Returns: + bytes: + Raw xlsx bytes. + + Raises: + Exception: + Retrieval-specific errors defined by the implementation. + """ + raise NotImplementedError + + +class FileSystemXlsxClient(BaseXlsxClient): + """ + XLSX client that reads from the local filesystem. + + Notes: + **Guarantees:** + + - This client reads spreadsheet files directly from the disk and + returns their raw binary contents. + """ + + def fetch(self, path: Path) -> bytes: + """ + Read an xlsx file from the local filesystem. + + Args: + path (Path): + Filesystem path to the spreadsheet file. + + Returns: + bytes: + Raw xlsx bytes. + + Raises: + FileNotFoundError: + If the path does not exist. + ValueError: + If the path exists but is not a file. + """ + + if not path.exists(): + raise FileNotFoundError(f"XLSX not found: {path}") + + if not path.is_file(): + raise ValueError(f"Path is not a file: {path}") + + return path.read_bytes() diff --git a/omniread/xlsx/parser.py b/omniread/xlsx/parser.py new file mode 100644 index 0000000..bb70288 --- /dev/null +++ b/omniread/xlsx/parser.py @@ -0,0 +1,146 @@ +""" +# Summary + +XLSX parser implementations for OmniRead. + +This module provides a concrete, generic parser for Office Open XML +spreadsheets. It exposes workbook sheets as lists of string rows so +downstream consumers can interpret tabular content without depending on +openpyxl directly. + +The parser is intentionally statement-agnostic: it performs no header +detection or column interpretation beyond basic cell normalization. +""" + +import datetime +from io import BytesIO +from typing import List, Optional, Union + +import openpyxl + +from omniread.core.content import Content +from .parser_base import XlsxParserBase + + +class XlsxParser(XlsxParserBase): + """ + Generic xlsx parser producing string rows from a worksheet. + + Notes: + **Responsibilities:** + + - Lazily load the workbook owned by the parser's content. + - Normalize cells (including dates and numeric values) into + deterministic string representations. + - Expose sheet discovery and row extraction helpers. + + **Constraints:** + + - Cells are rendered with ``str(value)`` after trimming; date and + datetime values are rendered in ISO format. Consumers requiring + locale-specific formatting must convert on their side. + """ + + def __init__(self, content: Content, *, data_only: bool = True, read_only: bool = True): + """ + Initialize the parser. + + Args: + content (Content): + XLSX content to parse; its type must be supported. + data_only (bool): + Passed to openpyxl: when True, formula cells yield their last + computed value instead of the formula string. + read_only (bool): + Passed to openpyxl: streaming mode for lower memory usage. + """ + super().__init__(content) + self._data_only = data_only + self._read_only = read_only + self._workbook: Optional[openpyxl.Workbook] = None + + @property + def workbook(self) -> openpyxl.Workbook: + """ + The lazily loaded workbook backing this parser's content. + """ + if self._workbook is None: + self._workbook = openpyxl.load_workbook( + BytesIO(self.content.raw), + data_only=self._data_only, + read_only=self._read_only, + ) + return self._workbook + + @property + def sheet_names(self) -> List[str]: + """ + Names of all worksheets contained in the workbook. + """ + return list(self.workbook.sheetnames) + + def parse(self) -> List[List[str]]: + """ + Parse the first worksheet into normalized string rows. + + Returns: + List[List[str]]: + Rows of the default (first) worksheet. + """ + return self.rows() + + def rows( + self, + sheet: Optional[Union[int, str]] = None, + *, + skip_empty: bool = True, + ) -> List[List[str]]: + """ + Extract normalized string rows from a worksheet. + + Args: + sheet (Optional[Union[int, str]]): + Worksheet index or title; defaults to the first worksheet. + skip_empty (bool): + When True (default), rows whose cells are all blank are omitted. + + Returns: + List[List[str]]: + Normalized rows; trailing blank cells are trimmed per row. + + Raises: + ValueError: + If the requested sheet does not exist. + """ + ws = self._resolve_sheet(sheet) + out: List[List[str]] = [] + for row in ws.iter_rows(values_only=True): + cells = [self._cell_str(v) for v in row] + while cells and not cells[-1].strip(): + cells.pop() + if skip_empty and not any(c.strip() for c in cells): + continue + out.append(cells) + return out + + def _resolve_sheet(self, sheet: Optional[Union[int, str]]): + names = self.workbook.sheetnames + if not names: + raise ValueError("Workbook contains no worksheets") + if sheet is None: + index = 0 + elif isinstance(sheet, int): + index = sheet + else: + index = names.index(sheet) if sheet in names else -1 + if not (0 <= index < len(names)): + raise ValueError(f"Worksheet not found: {sheet!r} (available: {names})") + return self.workbook[names[index]] + + @staticmethod + def _cell_str(value) -> str: + if value is None: + return "" + if isinstance(value, (datetime.datetime, datetime.date)): + return value.isoformat() + return str(value).strip() diff --git a/omniread/xlsx/parser_base.py b/omniread/xlsx/parser_base.py new file mode 100644 index 0000000..9974ef6 --- /dev/null +++ b/omniread/xlsx/parser_base.py @@ -0,0 +1,55 @@ +""" +# Summary + +XLSX parser base implementation for OmniRead. + +This module defines the **XLSX-specific parser contract**, extending the +format-agnostic `BaseParser` with constraints appropriate for Office Open +XML spreadsheet content. +""" + +from typing import Generic, TypeVar +from abc import abstractmethod + +from omniread.core.content import ContentType +from omniread.core.parser import BaseParser + +T = TypeVar("T") + + +class XlsxParserBase(BaseParser[T], Generic[T]): + """ + Base xlsx parser. + + Notes: + **Responsibilities:** + + - This class enforces xlsx content-type compatibility and provides + the extension point for implementing concrete xlsx parsing + strategies. + + **Constraints:** + + - Concrete implementations must define the output type `T` and + implement the `parse()` method. + """ + + supported_types = {ContentType.XLSX} + """ + Set of content types supported by this parser (XLSX only). + """ + + @abstractmethod + def parse(self) -> T: + """ + Parse xlsx content into a structured output. + + Returns: + T: + Parsed representation of type `T`. + + Raises: + Exception: + Parsing-specific errors as defined by the implementation. + """ + raise NotImplementedError diff --git a/omniread/xlsx/scraper.py b/omniread/xlsx/scraper.py new file mode 100644 index 0000000..00202ad --- /dev/null +++ b/omniread/xlsx/scraper.py @@ -0,0 +1,78 @@ +""" +# Summary + +XLSX scraper for OmniRead. + +This module defines the scraper responsible for acquiring raw Office Open +XML spreadsheet content from a backing store via a configured client. + +The scraper does not interpret or parse the acquired bytes; it wraps them in +the canonical `Content` model. +""" + +from typing import Any, Mapping, Optional + +from omniread.core.content import Content, ContentType +from .client import BaseXlsxClient + + +class XlsxScraper: + """ + Scraper for xlsx spreadsheet documents. + + Notes: + **Responsibilities:** + + - Fetch raw xlsx bytes via the configured client. + - Wrap the payload in a canonical `Content` instance with the + XLSX content type and source identifier. + + **Constraints:** + + - The scraper does not perform parsing or interpretation. + - Does not assume a specific storage backend. + """ + + def __init__(self, *, client: BaseXlsxClient): + """ + Initialize the XLSX scraper. + + Args: + client (BaseXlsxClient): + Client responsible for retrieving raw spreadsheet bytes. + """ + self._client = client + + def fetch( + self, + source: Any, + *, + metadata: Optional[Mapping[str, Any]] = None, + ) -> Content: + """ + Fetch an xlsx document from the given source. + + Args: + source (Any): + Identifier of the spreadsheet source as understood by the + configured client. + metadata (Optional[Mapping[str, Any]], optional): + Optional metadata to attach to the returned content. + + Returns: + Content: + A `Content` instance containing raw xlsx bytes, source + identifier, XLSX content type, and optional metadata. + + Raises: + Exception: + Retrieval-specific errors raised by the client. + """ + raw = self._client.fetch(source) + + return Content( + raw=raw, + source=source, + content_type=ContentType.XLSX, + metadata=dict(metadata) if metadata else None, + ) diff --git a/pyproject.toml b/pyproject.toml index 0219d36..fb78aff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ dependencies = [ "beautifulsoup4>=4.12.0", # "lxml>=5.0.0", "pypdf>=4.0.0", + "openpyxl>=3.1.0", ] [project.optional-dependencies] diff --git a/tests/test_xlsx_simple.py b/tests/test_xlsx_simple.py new file mode 100644 index 0000000..1390b19 --- /dev/null +++ b/tests/test_xlsx_simple.py @@ -0,0 +1,118 @@ +import datetime +from io import BytesIO + +import openpyxl +import pytest + +from omniread import ( + # core + Content, + ContentType, + + # xlsx + FileSystemXlsxClient, + XlsxParser, + XlsxScraper, +) + + +def _fixture_xlsx_bytes() -> bytes: + wb = openpyxl.Workbook() + ws = wb.active + ws.title = "Statement" + ws.append(["DETAILED STATEMENT", None, None, ""]) + ws.append([]) + ws.append(["S No", "Value Date", "Txn Date", "Remarks", "Withdrawal", "Deposit", "Balance"]) + ws.append([1, "01/06/2026", "01/06/2026", "UPI/NPCI BHIM/bhimcashback@h/B", 0.0, 10.0, 250285.24]) + row = 4 + ws.cell(row=row, column=5).value = 100.0 # ensure float rendering path + buf = BytesIO() + wb.save(buf) + return buf.getvalue() + + +@pytest.fixture +def xlsx_bytes() -> bytes: + return _fixture_xlsx_bytes() + + +@pytest.fixture +def xlsx_path(tmp_path, xlsx_bytes): + p = tmp_path / "statement.xlsx" + p.write_bytes(xlsx_bytes) + return p + + +def test_content_type_value(): + assert ContentType.XLSX.value == ( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ) + + +def test_scraper_round_trip(xlsx_path, xlsx_bytes): + scraper = XlsxScraper(client=FileSystemXlsxClient()) + content = scraper.fetch(xlsx_path) + + assert content.raw == xlsx_bytes + assert content.content_type is ContentType.XLSX + + +def test_client_missing_file_raises(tmp_path): + with pytest.raises(FileNotFoundError): + FileSystemXlsxClient().fetch(tmp_path / "nope.xlsx") + + +def test_client_directory_raises(tmp_path): + with pytest.raises(ValueError): + FileSystemXlsxClient().fetch(tmp_path) + + +def test_parser_rows_skip_empty_and_trim(xlsx_bytes): + parser = XlsxParser(Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX)) + + rows = parser.rows() + + # empty row dropped by skip_empty; metadata + header + data remain + assert len(rows) == 3 + assert rows[0] == ["DETAILED STATEMENT"] + header = rows[1] + assert header[:7] == ["S No", "Value Date", "Txn Date", "Remarks", "Withdrawal", "Deposit", "Balance"] + data = rows[2] + assert data[3] == "UPI/NPCI BHIM/bhimcashback@h/B" + assert data[6] == "250285.24" + + +def test_parser_keep_empty_rows(xlsx_bytes): + parser = XlsxParser(Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX)) + + rows = parser.rows(skip_empty=False) + + assert len(rows) == 4 # includes blank second row and padded metadata row + + +def test_parser_sheet_selection_and_names(xlsx_bytes): + parser = XlsxParser(Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX)) + + assert parser.sheet_names == ["Statement"] + assert parser.rows(sheet="Statement") == parser.rows(sheet=0) + assert parser.parse() == parser.rows() + + with pytest.raises(ValueError, match="not found"): + parser.rows(sheet="Missing") + + +def test_parser_date_cell_isoformat(): + wb = openpyxl.Workbook() + ws = wb.active + ws.append([datetime.date(2026, 6, 1), "x"]) + buf = BytesIO() + wb.save(buf) + + parser = XlsxParser(Content(raw=buf.getvalue(), source="mem", content_type=ContentType.XLSX)) + # openpyxl surfaces date cells as datetime; ISO rendering keeps determinism + assert parser.parse()[0][0] == "2026-06-01T00:00:00" + + +def test_parser_rejects_non_xlsx_content(): + with pytest.raises(ValueError, match="does not support"): + XlsxParser(Content(raw=b"", source="mem", content_type=ContentType.HTML))