mcp docs
This commit is contained in:
@@ -2,7 +2,7 @@
|
||||
"module": "omniread.html.scraper",
|
||||
"content": {
|
||||
"path": "omniread.html.scraper",
|
||||
"docstring": "HTML scraping implementation for OmniRead.\n\nThis module provides an HTTP-based scraper for retrieving HTML documents.\nIt implements the core `BaseScraper` contract using `httpx` as the transport\nlayer.\n\nThis scraper is responsible for:\n- Fetching raw HTML bytes over HTTP(S)\n- Validating response content type\n- Attaching HTTP metadata to the returned content\n\nThis scraper is not responsible for:\n- Parsing or interpreting HTML\n- Retrying failed requests\n- Managing crawl policies or rate limiting",
|
||||
"docstring": "HTML scraping implementation for OmniRead.\n\n---\n\n## Summary\n\nThis module provides an HTTP-based scraper for retrieving HTML documents.\nIt implements the core `BaseScraper` contract using `httpx` as the transport\nlayer.\n\nThis scraper is responsible for:\n- Fetching raw HTML bytes over HTTP(S)\n- Validating response content type\n- Attaching HTTP metadata to the returned content\n\nThis scraper is not responsible for:\n- Parsing or interpreting HTML\n- Retrying failed requests\n- Managing crawl policies or rate limiting",
|
||||
"objects": {
|
||||
"httpx": {
|
||||
"name": "httpx",
|
||||
@@ -37,35 +37,35 @@
|
||||
"kind": "class",
|
||||
"path": "omniread.html.scraper.Content",
|
||||
"signature": "<bound method Alias.signature of Alias('Content', 'omniread.core.content.Content')>",
|
||||
"docstring": "Normalized representation of extracted content.\n\nA `Content` instance represents a raw content payload along with minimal\ncontextual metadata describing its origin and type.\n\nThis class is the **primary exchange format** between:\n- Scrapers\n- Parsers\n- Downstream consumers\n\nAttributes:\n raw: Raw content bytes as retrieved from the source.\n source: Identifier of the content origin (URL, file path, or logical name).\n content_type: Optional MIME type of the content, if known.\n metadata: Optional, implementation-defined metadata associated with\n the content (e.g., headers, encoding hints, extraction notes).",
|
||||
"docstring": "Normalized representation of extracted content.\n\nNotes:\n **Responsibilities:**\n\n - A `Content` instance represents a raw content payload along with minimal contextual metadata describing its origin and type\n - This class is the primary exchange format between Scrapers, Parsers, and Downstream consumers",
|
||||
"members": {
|
||||
"raw": {
|
||||
"name": "raw",
|
||||
"kind": "attribute",
|
||||
"path": "omniread.html.scraper.Content.raw",
|
||||
"signature": "<bound method Alias.signature of Alias('raw', 'omniread.core.content.Content.raw')>",
|
||||
"docstring": null
|
||||
"docstring": "Raw content bytes as retrieved from the source."
|
||||
},
|
||||
"source": {
|
||||
"name": "source",
|
||||
"kind": "attribute",
|
||||
"path": "omniread.html.scraper.Content.source",
|
||||
"signature": "<bound method Alias.signature of Alias('source', 'omniread.core.content.Content.source')>",
|
||||
"docstring": null
|
||||
"docstring": "Identifier of the content origin (URL, file path, or logical name)."
|
||||
},
|
||||
"content_type": {
|
||||
"name": "content_type",
|
||||
"kind": "attribute",
|
||||
"path": "omniread.html.scraper.Content.content_type",
|
||||
"signature": "<bound method Alias.signature of Alias('content_type', 'omniread.core.content.Content.content_type')>",
|
||||
"docstring": null
|
||||
"docstring": "Optional MIME type of the content, if known."
|
||||
},
|
||||
"metadata": {
|
||||
"name": "metadata",
|
||||
"kind": "attribute",
|
||||
"path": "omniread.html.scraper.Content.metadata",
|
||||
"signature": "<bound method Alias.signature of Alias('metadata', 'omniread.core.content.Content.metadata')>",
|
||||
"docstring": null
|
||||
"docstring": "Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes)."
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -74,7 +74,7 @@
|
||||
"kind": "class",
|
||||
"path": "omniread.html.scraper.ContentType",
|
||||
"signature": "<bound method Alias.signature of Alias('ContentType', 'omniread.core.content.ContentType')>",
|
||||
"docstring": "Supported MIME types for extracted content.\n\nThis enum represents the declared or inferred media type of the content\nsource. It is primarily used for routing content to the appropriate\nparser or downstream consumer.",
|
||||
"docstring": "Supported MIME types for extracted content.\n\nNotes:\n **Guarantees:**\n\n - This enum represents the declared or inferred media type of the content source\n - It is primarily used for routing content to the appropriate parser or downstream consumer",
|
||||
"members": {
|
||||
"HTML": {
|
||||
"name": "HTML",
|
||||
@@ -111,14 +111,14 @@
|
||||
"kind": "class",
|
||||
"path": "omniread.html.scraper.BaseScraper",
|
||||
"signature": "<bound method Alias.signature of Alias('BaseScraper', 'omniread.core.scraper.BaseScraper')>",
|
||||
"docstring": "Base interface for all scrapers.\n\nA scraper is responsible ONLY for fetching raw content\n(bytes) from a source. It must not interpret or parse it.\n\nA scraper is a **stateless acquisition component** that retrieves raw\ncontent from a source and returns it as a `Content` object.\n\nScrapers define *how content is obtained*, not *what the content means*.\n\nImplementations may vary in:\n- Transport mechanism (HTTP, filesystem, cloud storage)\n- Authentication strategy\n- Retry and backoff behavior\n\nImplementations must not:\n- Parse content\n- Modify content semantics\n- Couple scraping logic to a specific parser",
|
||||
"docstring": "Base interface for all scrapers.\n\nNotes:\n **Responsibilities:**\n\n - A scraper is responsible ONLY for fetching raw content (bytes) from a source. It must not interpret or parse it\n - A scraper is a stateless acquisition component that retrieves raw content from a source and returns it as a `Content` object\n - Scrapers define how content is obtained, not what the content means\n - Implementations may vary in transport mechanism, authentication strategy, retry and backoff behavior\n\n **Constraints:**\n\n - Implementations must not parse content, modify content semantics, or couple scraping logic to a specific parser",
|
||||
"members": {
|
||||
"fetch": {
|
||||
"name": "fetch",
|
||||
"kind": "function",
|
||||
"path": "omniread.html.scraper.BaseScraper.fetch",
|
||||
"signature": "<bound method Alias.signature of Alias('fetch', 'omniread.core.scraper.BaseScraper.fetch')>",
|
||||
"docstring": "Fetch raw content from the given source.\n\nImplementations must retrieve the content referenced by `source`\nand return it as raw bytes wrapped in a `Content` object.\n\nArgs:\n source: Location identifier (URL, file path, S3 URI, etc.)\n metadata: Optional hints for the scraper (headers, auth, etc.)\n\nReturns:\n Content object containing raw bytes and metadata.\n - Raw content bytes\n - Source identifier\n - Optional metadata\n\nRaises:\n Exception: Retrieval-specific errors as defined by the implementation."
|
||||
"docstring": "Fetch raw content from the given source.\n\nArgs:\n source (str):\n Location identifier (URL, file path, S3 URI, etc.)\n metadata (Optional[Mapping[str, Any]], optional):\n Optional hints for the scraper (headers, auth, etc.)\n\nReturns:\n Content:\n Content object containing raw bytes and metadata.\n\nRaises:\n Exception:\n Retrieval-specific errors as defined by the implementation.\n\nNotes:\n **Responsibilities:**\n\n - Implementations must retrieve the content referenced by `source` and return it as raw bytes wrapped in a `Content` object"
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -126,8 +126,8 @@
|
||||
"name": "HTMLScraper",
|
||||
"kind": "class",
|
||||
"path": "omniread.html.scraper.HTMLScraper",
|
||||
"signature": "<bound method Class.signature of Class('HTMLScraper', 26, 134)>",
|
||||
"docstring": "Base HTML scraper using httpx.\n\nThis scraper retrieves HTML documents over HTTP(S) and returns them\nas raw content wrapped in a `Content` object.\n\nFetches raw bytes and metadata only.\nThe scraper:\n- Uses `httpx.Client` for HTTP requests\n- Enforces an HTML content type\n- Preserves HTTP response metadata\n\nThe scraper does not:\n- Parse HTML\n- Perform retries or backoff\n- Handle non-HTML responses",
|
||||
"signature": "<bound method Class.signature of Class('HTMLScraper', 30, 139)>",
|
||||
"docstring": "Base HTML scraper using httpx.\n\nNotes:\n **Responsibilities:**\n\n - This scraper retrieves HTML documents over HTTP(S) and returns them as raw content wrapped in a `Content` object\n - Fetches raw bytes and metadata only. The scraper uses `httpx.Client` for HTTP requests, enforces an HTML content type, preserves HTTP response metadata\n\n **Constraints:**\n \n - The scraper does not: Parse HTML, perform retries or backoff, handle non-HTML responses",
|
||||
"members": {
|
||||
"content_type": {
|
||||
"name": "content_type",
|
||||
@@ -140,15 +140,15 @@
|
||||
"name": "validate_content_type",
|
||||
"kind": "function",
|
||||
"path": "omniread.html.scraper.HTMLScraper.validate_content_type",
|
||||
"signature": "<bound method Function.signature of Function('validate_content_type', 71, 94)>",
|
||||
"docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response: HTTP response returned by `httpx`.\n\nRaises:\n ValueError: If the `Content-Type` header is missing or does not\n indicate HTML content."
|
||||
"signature": "<bound method Function.signature of Function('validate_content_type', 74, 98)>",
|
||||
"docstring": "Validate that the HTTP response contains HTML content.\n\nArgs:\n response (httpx.Response):\n HTTP response returned by `httpx`.\n\nRaises:\n ValueError:\n If the `Content-Type` header is missing or does not indicate HTML content."
|
||||
},
|
||||
"fetch": {
|
||||
"name": "fetch",
|
||||
"kind": "function",
|
||||
"path": "omniread.html.scraper.HTMLScraper.fetch",
|
||||
"signature": "<bound method Function.signature of Function('fetch', 96, 134)>",
|
||||
"docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source: URL of the HTML document.\n metadata: Optional metadata to be merged into the returned content.\n\nReturns:\n A `Content` instance containing:\n - Raw HTML bytes\n - Source URL\n - HTML content type\n - HTTP response metadata\n\nRaises:\n httpx.HTTPError: If the HTTP request fails.\n ValueError: If the response is not valid HTML."
|
||||
"signature": "<bound method Function.signature of Function('fetch', 100, 139)>",
|
||||
"docstring": "Fetch an HTML document from the given source.\n\nArgs:\n source (str):\n URL of the HTML document.\n metadata (Optional[Mapping[str, Any]], optional):\n Optional metadata to be merged into the returned content.\n\nReturns:\n Content:\n A `Content` instance containing raw HTML bytes, source URL, HTML content type, and HTTP response metadata.\n\nRaises:\n httpx.HTTPError:\n If the HTTP request fails.\n ValueError:\n If the response is not valid HTML."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user