omniread/mcp_docs/modules/omniread.core.scraper.json

{
  "module": "omniread.core.scraper",
  "content": {
    "path": "omniread.core.scraper",
    "docstring": "Abstract scraping contracts for OmniRead.\n\nThis module defines the **format-agnostic scraper interface** responsible for\nacquiring raw content from external sources.\n\nScrapers are responsible for:\n- Locating and retrieving raw content bytes\n- Attaching minimal contextual metadata\n- Returning normalized `Content` objects\n\nScrapers are explicitly NOT responsible for:\n- Parsing or interpreting content\n- Inferring structure or semantics\n- Performing content-type specific processing\n\nAll interpretation must be delegated to parsers.",
    "objects": {
      "ABC": {
        "name": "ABC",
        "kind": "alias",
        "path": "omniread.core.scraper.ABC",
        "signature": "<bound method Alias.signature of Alias('ABC', 'abc.ABC')>",
        "docstring": null
      },
      "abstractmethod": {
        "name": "abstractmethod",
        "kind": "alias",
        "path": "omniread.core.scraper.abstractmethod",
        "signature": "<bound method Alias.signature of Alias('abstractmethod', 'abc.abstractmethod')>",
        "docstring": null
      },
      "Any": {
        "name": "Any",
        "kind": "alias",
        "path": "omniread.core.scraper.Any",
        "signature": "<bound method Alias.signature of Alias('Any', 'typing.Any')>",
        "docstring": null
      },
      "Mapping": {
        "name": "Mapping",
        "kind": "alias",
        "path": "omniread.core.scraper.Mapping",
        "signature": "<bound method Alias.signature of Alias('Mapping', 'typing.Mapping')>",
        "docstring": null
      },
      "Optional": {
        "name": "Optional",
        "kind": "alias",
        "path": "omniread.core.scraper.Optional",
        "signature": "<bound method Alias.signature of Alias('Optional', 'typing.Optional')>",
        "docstring": null
      },
      "Content": {
        "name": "Content",
        "kind": "class",
        "path": "omniread.core.scraper.Content",
        "signature": "<bound method Alias.signature of Alias('Content', 'omniread.core.content.Content')>",
        "docstring": "Normalized representation of extracted content.\n\nA `Content` instance represents a raw content payload along with minimal\ncontextual metadata describing its origin and type.\n\nThis class is the **primary exchange format** between:\n- Scrapers\n- Parsers\n- Downstream consumers\n\nAttributes:\n    raw: Raw content bytes as retrieved from the source.\n    source: Identifier of the content origin (URL, file path, or logical name).\n    content_type: Optional MIME type of the content, if known.\n    metadata: Optional, implementation-defined metadata associated with\n        the content (e.g., headers, encoding hints, extraction notes).",
        "members": {
          "raw": {
            "name": "raw",
            "kind": "attribute",
            "path": "omniread.core.scraper.Content.raw",
            "signature": "<bound method Alias.signature of Alias('raw', 'omniread.core.content.Content.raw')>",
            "docstring": null
          },
          "source": {
            "name": "source",
            "kind": "attribute",
            "path": "omniread.core.scraper.Content.source",
            "signature": "<bound method Alias.signature of Alias('source', 'omniread.core.content.Content.source')>",
            "docstring": null
          },
          "content_type": {
            "name": "content_type",
            "kind": "attribute",
            "path": "omniread.core.scraper.Content.content_type",
            "signature": "<bound method Alias.signature of Alias('content_type', 'omniread.core.content.Content.content_type')>",
            "docstring": null
          },
          "metadata": {
            "name": "metadata",
            "kind": "attribute",
            "path": "omniread.core.scraper.Content.metadata",
            "signature": "<bound method Alias.signature of Alias('metadata', 'omniread.core.content.Content.metadata')>",
            "docstring": null
          }
        }
      },
      "BaseScraper": {
        "name": "BaseScraper",
        "kind": "class",
        "path": "omniread.core.scraper.BaseScraper",
        "signature": "<bound method Class.signature of Class('BaseScraper', 26, 75)>",
        "docstring": "Base interface for all scrapers.\n\nA scraper is responsible ONLY for fetching raw content\n(bytes) from a source. It must not interpret or parse it.\n\nA scraper is a **stateless acquisition component** that retrieves raw\ncontent from a source and returns it as a `Content` object.\n\nScrapers define *how content is obtained*, not *what the content means*.\n\nImplementations may vary in:\n- Transport mechanism (HTTP, filesystem, cloud storage)\n- Authentication strategy\n- Retry and backoff behavior\n\nImplementations must not:\n- Parse content\n- Modify content semantics\n- Couple scraping logic to a specific parser",
        "members": {
          "fetch": {
            "name": "fetch",
            "kind": "function",
            "path": "omniread.core.scraper.BaseScraper.fetch",
            "signature": "<bound method Function.signature of Function('fetch', 49, 75)>",
            "docstring": "Fetch raw content from the given source.\n\nImplementations must retrieve the content referenced by `source`\nand return it as raw bytes wrapped in a `Content` object.\n\nArgs:\n    source: Location identifier (URL, file path, S3 URI, etc.)\n    metadata: Optional hints for the scraper (headers, auth, etc.)\n\nReturns:\n    Content object containing raw bytes and metadata.\n    - Raw content bytes\n    - Source identifier\n    - Optional metadata\n\nRaises:\n    Exception: Retrieval-specific errors as defined by the implementation."
          }
        }
      }
    }
  }
}