using doc-forge (#1)

Reviewed-on: #1
Co-authored-by: Vishesh 'ironeagle' Bangotra <aetoskia@gmail.com>
Co-committed-by: Vishesh 'ironeagle' Bangotra <aetoskia@gmail.com>
This commit is contained in:
2026-01-22 11:27:56 +00:00
committed by aetos
parent 6808538485
commit 67a3074ab4
46 changed files with 4475 additions and 107 deletions

View File

@@ -94,6 +94,27 @@ PDF:
- FileSystemPDFClient
- PDFScraper
- PDFParser
## Core Philosophy
`OmniRead` is designed as a **decoupled content engine**:
1. **Separation of Concerns**: Scrapers *fetch*, Parsers *interpret*. Neither knows about the other.
2. **Normalized Exchange**: All components communicate via the `Content` model, ensuring a consistent contract.
3. **Format Agnosticism**: The core logic is independent of whether the input is HTML, PDF, or JSON.
## Documentation Design
For those extending `OmniRead`, follow these "AI-Native" docstring principles:
### For Humans
- **Clear Contracts**: Explicitly state what a component is and is NOT responsible for.
- **Runnable Examples**: Include small, logical snippets in the package `__init__.py`.
### For LLMs
- **Structured Models**: Use dataclasses and enums for core data to ensure clean MCP JSON representation.
- **Type Safety**: All public APIs must be fully typed and have corresponding `.pyi` stubs.
- **Detailed Raises**: Include `: description` pairs in the `Raises` section to help agents handle errors gracefully.
"""
from .core import Content, ContentType

13
omniread/__init__.pyi Normal file
View File

@@ -0,0 +1,13 @@
from .core import Content, ContentType
from .html import HTMLScraper, HTMLParser
from .pdf import FileSystemPDFClient, PDFScraper, PDFParser
__all__ = [
"Content",
"ContentType",
"HTMLScraper",
"HTMLParser",
"FileSystemPDFClient",
"PDFScraper",
"PDFParser",
]

View File

@@ -0,0 +1,10 @@
from .content import Content, ContentType
from .parser import BaseParser
from .scraper import BaseScraper
__all__ = [
"Content",
"ContentType",
"BaseParser",
"BaseScraper",
]

15
omniread/core/content.pyi Normal file
View File

@@ -0,0 +1,15 @@
from enum import Enum
from typing import Any, Mapping, Optional
class ContentType(str, Enum):
HTML = "text/html"
PDF = "application/pdf"
JSON = "application/json"
XML = "application/xml"
class Content:
raw: bytes
source: str
content_type: Optional[ContentType]
metadata: Optional[Mapping[str, Any]]
def __init__(self, raw: bytes, source: str, content_type: Optional[ContentType] = ..., metadata: Optional[Mapping[str, Any]] = ...) -> None: ...

13
omniread/core/parser.pyi Normal file
View File

@@ -0,0 +1,13 @@
from abc import ABC, abstractmethod
from typing import Generic, TypeVar, Set
from .content import Content, ContentType
T = TypeVar("T")
class BaseParser(ABC, Generic[T]):
supported_types: Set[ContentType]
content: Content
def __init__(self, content: Content) -> None: ...
@abstractmethod
def parse(self) -> T: ...
def supports(self) -> bool: ...

View File

@@ -0,0 +1,7 @@
from abc import ABC, abstractmethod
from typing import Any, Mapping, Optional
from .content import Content
class BaseScraper(ABC):
@abstractmethod
def fetch(self, source: str, *, metadata: Optional[Mapping[str, Any]] = ...) -> Content: ...

View File

@@ -0,0 +1,4 @@
from .scraper import HTMLScraper
from .parser import HTMLParser
__all__ = ["HTMLScraper", "HTMLParser"]

18
omniread/html/parser.pyi Normal file
View File

@@ -0,0 +1,18 @@
from typing import Any, Generic, TypeVar, Optional, list, dict
from bs4 import BeautifulSoup, Tag
from omniread.core.content import ContentType, Content
from omniread.core.parser import BaseParser
T = TypeVar("T")
class HTMLParser(BaseParser[T], Generic[T]):
supported_types: set[ContentType]
def __init__(self, content: Content, features: str = ...) -> None: ...
def parse(self) -> T: ...
@staticmethod
def parse_div(div: Tag, *, separator: str = ...) -> str: ...
@staticmethod
def parse_link(a: Tag) -> Optional[str]: ...
@staticmethod
def parse_table(table: Tag) -> list[list[str]]: ...
def parse_meta(self) -> dict[str, Any]: ...

10
omniread/html/scraper.pyi Normal file
View File

@@ -0,0 +1,10 @@
import httpx
from typing import Any, Mapping, Optional
from omniread.core.content import Content, ContentType
from omniread.core.scraper import BaseScraper
class HTMLScraper(BaseScraper):
content_type: ContentType
def __init__(self, *, client: Optional[httpx.Client] = ..., timeout: float = ..., headers: Optional[Mapping[str, str]] = ..., follow_redirects: bool = ...) -> None: ...
def validate_content_type(self, response: httpx.Response) -> None: ...
def fetch(self, source: str, *, metadata: Optional[Mapping[str, Any]] = ...) -> Content: ...

View File

@@ -0,0 +1,5 @@
from .client import FileSystemPDFClient
from .scraper import PDFScraper
from .parser import PDFParser
__all__ = ["FileSystemPDFClient", "PDFScraper", "PDFParser"]

View File

@@ -14,6 +14,7 @@ Typical backing stores include:
- Network file systems
"""
from typing import Any
from abc import ABC, abstractmethod
from pathlib import Path
@@ -30,7 +31,7 @@ class BasePDFClient(ABC):
"""
@abstractmethod
def fetch(self, source: str) -> bytes:
def fetch(self, source: Any) -> bytes:
"""
Fetch raw PDF bytes from the given source.

10
omniread/pdf/client.pyi Normal file
View File

@@ -0,0 +1,10 @@
from abc import ABC, abstractmethod
from pathlib import Path
from typing import Any
class BasePDFClient(ABC):
@abstractmethod
def fetch(self, source: Any) -> bytes: ...
class FileSystemPDFClient(BasePDFClient):
def fetch(self, source: Path | str) -> bytes: ...

11
omniread/pdf/parser.pyi Normal file
View File

@@ -0,0 +1,11 @@
from abc import abstractmethod
from typing import Generic, TypeVar
from omniread.core.content import ContentType
from omniread.core.parser import BaseParser
T = TypeVar("T")
class PDFParser(BaseParser[T], Generic[T]):
supported_types: set[ContentType]
@abstractmethod
def parse(self) -> T: ...

View File

@@ -39,7 +39,7 @@ class PDFScraper(BaseScraper):
def fetch(
self,
source: str,
source: Any,
*,
metadata: Optional[Mapping[str, Any]] = None,
) -> Content:

8
omniread/pdf/scraper.pyi Normal file
View File

@@ -0,0 +1,8 @@
from typing import Any, Mapping, Optional
from omniread.core.content import Content, ContentType
from omniread.core.scraper import BaseScraper
from .client import BasePDFClient
class PDFScraper(BaseScraper):
def __init__(self, *, client: BasePDFClient) -> None: ...
def fetch(self, source: Any, *, metadata: Optional[Mapping[str, Any]] = ...) -> Content: ...