using doc-forge (#1)
Reviewed-on: #1 Co-authored-by: Vishesh 'ironeagle' Bangotra <aetoskia@gmail.com> Co-committed-by: Vishesh 'ironeagle' Bangotra <aetoskia@gmail.com>
This commit is contained in:
@@ -94,6 +94,27 @@ PDF:
|
||||
- FileSystemPDFClient
|
||||
- PDFScraper
|
||||
- PDFParser
|
||||
|
||||
## Core Philosophy
|
||||
|
||||
`OmniRead` is designed as a **decoupled content engine**:
|
||||
|
||||
1. **Separation of Concerns**: Scrapers *fetch*, Parsers *interpret*. Neither knows about the other.
|
||||
2. **Normalized Exchange**: All components communicate via the `Content` model, ensuring a consistent contract.
|
||||
3. **Format Agnosticism**: The core logic is independent of whether the input is HTML, PDF, or JSON.
|
||||
|
||||
## Documentation Design
|
||||
|
||||
For those extending `OmniRead`, follow these "AI-Native" docstring principles:
|
||||
|
||||
### For Humans
|
||||
- **Clear Contracts**: Explicitly state what a component is and is NOT responsible for.
|
||||
- **Runnable Examples**: Include small, logical snippets in the package `__init__.py`.
|
||||
|
||||
### For LLMs
|
||||
- **Structured Models**: Use dataclasses and enums for core data to ensure clean MCP JSON representation.
|
||||
- **Type Safety**: All public APIs must be fully typed and have corresponding `.pyi` stubs.
|
||||
- **Detailed Raises**: Include `: description` pairs in the `Raises` section to help agents handle errors gracefully.
|
||||
"""
|
||||
|
||||
from .core import Content, ContentType
|
||||
|
||||
13
omniread/__init__.pyi
Normal file
13
omniread/__init__.pyi
Normal file
@@ -0,0 +1,13 @@
|
||||
from .core import Content, ContentType
|
||||
from .html import HTMLScraper, HTMLParser
|
||||
from .pdf import FileSystemPDFClient, PDFScraper, PDFParser
|
||||
|
||||
__all__ = [
|
||||
"Content",
|
||||
"ContentType",
|
||||
"HTMLScraper",
|
||||
"HTMLParser",
|
||||
"FileSystemPDFClient",
|
||||
"PDFScraper",
|
||||
"PDFParser",
|
||||
]
|
||||
10
omniread/core/__init__.pyi
Normal file
10
omniread/core/__init__.pyi
Normal file
@@ -0,0 +1,10 @@
|
||||
from .content import Content, ContentType
|
||||
from .parser import BaseParser
|
||||
from .scraper import BaseScraper
|
||||
|
||||
__all__ = [
|
||||
"Content",
|
||||
"ContentType",
|
||||
"BaseParser",
|
||||
"BaseScraper",
|
||||
]
|
||||
15
omniread/core/content.pyi
Normal file
15
omniread/core/content.pyi
Normal file
@@ -0,0 +1,15 @@
|
||||
from enum import Enum
|
||||
from typing import Any, Mapping, Optional
|
||||
|
||||
class ContentType(str, Enum):
|
||||
HTML = "text/html"
|
||||
PDF = "application/pdf"
|
||||
JSON = "application/json"
|
||||
XML = "application/xml"
|
||||
|
||||
class Content:
|
||||
raw: bytes
|
||||
source: str
|
||||
content_type: Optional[ContentType]
|
||||
metadata: Optional[Mapping[str, Any]]
|
||||
def __init__(self, raw: bytes, source: str, content_type: Optional[ContentType] = ..., metadata: Optional[Mapping[str, Any]] = ...) -> None: ...
|
||||
13
omniread/core/parser.pyi
Normal file
13
omniread/core/parser.pyi
Normal file
@@ -0,0 +1,13 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Generic, TypeVar, Set
|
||||
from .content import Content, ContentType
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
class BaseParser(ABC, Generic[T]):
|
||||
supported_types: Set[ContentType]
|
||||
content: Content
|
||||
def __init__(self, content: Content) -> None: ...
|
||||
@abstractmethod
|
||||
def parse(self) -> T: ...
|
||||
def supports(self) -> bool: ...
|
||||
7
omniread/core/scraper.pyi
Normal file
7
omniread/core/scraper.pyi
Normal file
@@ -0,0 +1,7 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, Mapping, Optional
|
||||
from .content import Content
|
||||
|
||||
class BaseScraper(ABC):
|
||||
@abstractmethod
|
||||
def fetch(self, source: str, *, metadata: Optional[Mapping[str, Any]] = ...) -> Content: ...
|
||||
4
omniread/html/__init__.pyi
Normal file
4
omniread/html/__init__.pyi
Normal file
@@ -0,0 +1,4 @@
|
||||
from .scraper import HTMLScraper
|
||||
from .parser import HTMLParser
|
||||
|
||||
__all__ = ["HTMLScraper", "HTMLParser"]
|
||||
18
omniread/html/parser.pyi
Normal file
18
omniread/html/parser.pyi
Normal file
@@ -0,0 +1,18 @@
|
||||
from typing import Any, Generic, TypeVar, Optional, list, dict
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
from omniread.core.content import ContentType, Content
|
||||
from omniread.core.parser import BaseParser
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
class HTMLParser(BaseParser[T], Generic[T]):
|
||||
supported_types: set[ContentType]
|
||||
def __init__(self, content: Content, features: str = ...) -> None: ...
|
||||
def parse(self) -> T: ...
|
||||
@staticmethod
|
||||
def parse_div(div: Tag, *, separator: str = ...) -> str: ...
|
||||
@staticmethod
|
||||
def parse_link(a: Tag) -> Optional[str]: ...
|
||||
@staticmethod
|
||||
def parse_table(table: Tag) -> list[list[str]]: ...
|
||||
def parse_meta(self) -> dict[str, Any]: ...
|
||||
10
omniread/html/scraper.pyi
Normal file
10
omniread/html/scraper.pyi
Normal file
@@ -0,0 +1,10 @@
|
||||
import httpx
|
||||
from typing import Any, Mapping, Optional
|
||||
from omniread.core.content import Content, ContentType
|
||||
from omniread.core.scraper import BaseScraper
|
||||
|
||||
class HTMLScraper(BaseScraper):
|
||||
content_type: ContentType
|
||||
def __init__(self, *, client: Optional[httpx.Client] = ..., timeout: float = ..., headers: Optional[Mapping[str, str]] = ..., follow_redirects: bool = ...) -> None: ...
|
||||
def validate_content_type(self, response: httpx.Response) -> None: ...
|
||||
def fetch(self, source: str, *, metadata: Optional[Mapping[str, Any]] = ...) -> Content: ...
|
||||
5
omniread/pdf/__init__.pyi
Normal file
5
omniread/pdf/__init__.pyi
Normal file
@@ -0,0 +1,5 @@
|
||||
from .client import FileSystemPDFClient
|
||||
from .scraper import PDFScraper
|
||||
from .parser import PDFParser
|
||||
|
||||
__all__ = ["FileSystemPDFClient", "PDFScraper", "PDFParser"]
|
||||
@@ -14,6 +14,7 @@ Typical backing stores include:
|
||||
- Network file systems
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
from abc import ABC, abstractmethod
|
||||
from pathlib import Path
|
||||
|
||||
@@ -30,7 +31,7 @@ class BasePDFClient(ABC):
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def fetch(self, source: str) -> bytes:
|
||||
def fetch(self, source: Any) -> bytes:
|
||||
"""
|
||||
Fetch raw PDF bytes from the given source.
|
||||
|
||||
|
||||
10
omniread/pdf/client.pyi
Normal file
10
omniread/pdf/client.pyi
Normal file
@@ -0,0 +1,10 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
class BasePDFClient(ABC):
|
||||
@abstractmethod
|
||||
def fetch(self, source: Any) -> bytes: ...
|
||||
|
||||
class FileSystemPDFClient(BasePDFClient):
|
||||
def fetch(self, source: Path | str) -> bytes: ...
|
||||
11
omniread/pdf/parser.pyi
Normal file
11
omniread/pdf/parser.pyi
Normal file
@@ -0,0 +1,11 @@
|
||||
from abc import abstractmethod
|
||||
from typing import Generic, TypeVar
|
||||
from omniread.core.content import ContentType
|
||||
from omniread.core.parser import BaseParser
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
class PDFParser(BaseParser[T], Generic[T]):
|
||||
supported_types: set[ContentType]
|
||||
@abstractmethod
|
||||
def parse(self) -> T: ...
|
||||
@@ -39,7 +39,7 @@ class PDFScraper(BaseScraper):
|
||||
|
||||
def fetch(
|
||||
self,
|
||||
source: str,
|
||||
source: Any,
|
||||
*,
|
||||
metadata: Optional[Mapping[str, Any]] = None,
|
||||
) -> Content:
|
||||
|
||||
8
omniread/pdf/scraper.pyi
Normal file
8
omniread/pdf/scraper.pyi
Normal file
@@ -0,0 +1,8 @@
|
||||
from typing import Any, Mapping, Optional
|
||||
from omniread.core.content import Content, ContentType
|
||||
from omniread.core.scraper import BaseScraper
|
||||
from .client import BasePDFClient
|
||||
|
||||
class PDFScraper(BaseScraper):
|
||||
def __init__(self, *, client: BasePDFClient) -> None: ...
|
||||
def fetch(self, source: Any, *, metadata: Optional[Mapping[str, Any]] = ...) -> Content: ...
|
||||
Reference in New Issue
Block a user