omni read basic modules
This commit is contained in:
0
omniread/__init__.py
Normal file
0
omniread/__init__.py
Normal file
0
omniread/core/__init__.py
Normal file
0
omniread/core/__init__.py
Normal file
18
omniread/core/content.py
Normal file
18
omniread/core/content.py
Normal file
@@ -0,0 +1,18 @@
|
||||
from enum import Enum
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Mapping, Optional
|
||||
|
||||
|
||||
class ContentType(str, Enum):
|
||||
HTML = "text/html"
|
||||
PDF = "application/pdf"
|
||||
JSON = "application/json"
|
||||
XML = "application/xml"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Content:
|
||||
raw: bytes
|
||||
source: str
|
||||
content_type: Optional[ContentType] = None
|
||||
metadata: Optional[Mapping[str, Any]] = None
|
||||
48
omniread/core/parser.py
Normal file
48
omniread/core/parser.py
Normal file
@@ -0,0 +1,48 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Generic, TypeVar, Set
|
||||
|
||||
from .content import Content, ContentType
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class BaseParser(ABC, Generic[T]):
|
||||
"""
|
||||
Base interface for all parsers.
|
||||
|
||||
A parser is a self-contained object that owns the Content
|
||||
it is responsible for interpreting.
|
||||
"""
|
||||
|
||||
supported_types: Set[ContentType] = set()
|
||||
|
||||
def __init__(self, content: Content):
|
||||
self.content = content
|
||||
|
||||
if not self.supports():
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} does not support content type "
|
||||
f"{content.content_type!r}"
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def parse(self) -> T:
|
||||
"""
|
||||
Parse the owned content into structured output.
|
||||
|
||||
Returns:
|
||||
Parsed, structured representation.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def supports(self) -> bool:
|
||||
"""
|
||||
Check whether this parser supports the content's type.
|
||||
"""
|
||||
if not self.supported_types:
|
||||
return True
|
||||
|
||||
if self.content.content_type is None:
|
||||
return False
|
||||
|
||||
return self.content.content_type in self.supported_types
|
||||
32
omniread/core/scraper.py
Normal file
32
omniread/core/scraper.py
Normal file
@@ -0,0 +1,32 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, Mapping, Optional
|
||||
|
||||
from .content import Content
|
||||
|
||||
|
||||
class BaseScraper(ABC):
|
||||
"""
|
||||
Base interface for all scrapers.
|
||||
|
||||
A scraper is responsible ONLY for fetching raw content
|
||||
(bytes) from a source. It must not interpret or parse it.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def fetch(
|
||||
self,
|
||||
source: str,
|
||||
*,
|
||||
metadata: Optional[Mapping[str, Any]] = None,
|
||||
) -> Content:
|
||||
"""
|
||||
Fetch raw content from the given source.
|
||||
|
||||
Args:
|
||||
source: Location identifier (URL, file path, S3 URI, etc.)
|
||||
metadata: Optional hints for the scraper (headers, auth, etc.)
|
||||
|
||||
Returns:
|
||||
Content object containing raw bytes and metadata.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
0
omniread/html/__init__.py
Normal file
0
omniread/html/__init__.py
Normal file
85
omniread/html/parser.py
Normal file
85
omniread/html/parser.py
Normal file
@@ -0,0 +1,85 @@
|
||||
from typing import Any, Generic, TypeVar, Optional
|
||||
|
||||
from abc import abstractmethod
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
from omniread.core.content import ContentType, Content
|
||||
from omniread.core.parser import BaseParser
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class HTMLParser(BaseParser[T], Generic[T]):
|
||||
"""
|
||||
Base HTML parser.
|
||||
|
||||
Provides reusable helpers for HTML extraction.
|
||||
Concrete parsers must explicitly define the return type.
|
||||
"""
|
||||
|
||||
supported_types = {ContentType.HTML}
|
||||
|
||||
def __init__(self, content: Content, features: str = "html.parser"):
|
||||
super().__init__(content)
|
||||
self._features = features
|
||||
self._soup = self._get_soup()
|
||||
|
||||
# ----------------------------
|
||||
# Contract
|
||||
# ----------------------------
|
||||
|
||||
@abstractmethod
|
||||
def parse(self) -> T:
|
||||
"""
|
||||
Fully parse the HTML content into structured output.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
# ----------------------------
|
||||
# Helpers (static / pure)
|
||||
# ----------------------------
|
||||
|
||||
@staticmethod
|
||||
def parse_div(div: Tag, *, separator: str = " ") -> str:
|
||||
return div.get_text(separator=separator, strip=True)
|
||||
|
||||
@staticmethod
|
||||
def parse_link(a: Tag) -> Optional[str]:
|
||||
return a.get("href")
|
||||
|
||||
@staticmethod
|
||||
def parse_table(table: Tag) -> list[list[str]]:
|
||||
rows: list[list[str]] = []
|
||||
for tr in table.find_all("tr"):
|
||||
cells = [
|
||||
cell.get_text(strip=True)
|
||||
for cell in tr.find_all(["td", "th"])
|
||||
]
|
||||
if cells:
|
||||
rows.append(cells)
|
||||
return rows
|
||||
|
||||
# ----------------------------
|
||||
# Helpers (instance-level)
|
||||
# ----------------------------
|
||||
|
||||
def _get_soup(self) -> BeautifulSoup:
|
||||
if not self.content.raw:
|
||||
raise ValueError("Empty HTML content")
|
||||
return BeautifulSoup(self.content.raw, features=self._features)
|
||||
|
||||
def parse_meta(self) -> dict[str, Any]:
|
||||
soup = self._soup
|
||||
|
||||
title = soup.title.string.strip() if soup.title and soup.title.string else None
|
||||
|
||||
meta = {
|
||||
tag.get("name") or tag.get("property"): tag.get("content")
|
||||
for tag in soup.find_all("meta")
|
||||
if tag.get("content")
|
||||
}
|
||||
|
||||
return {
|
||||
"title": title,
|
||||
"meta": meta,
|
||||
}
|
||||
48
omniread/html/scraper.py
Normal file
48
omniread/html/scraper.py
Normal file
@@ -0,0 +1,48 @@
|
||||
import httpx
|
||||
from typing import Any, Mapping, Optional
|
||||
|
||||
from omniread.core.content import Content
|
||||
from omniread.core.scraper import BaseScraper
|
||||
|
||||
|
||||
class HTMLScraper(BaseScraper):
|
||||
"""
|
||||
Base HTTP scraper using httpx.
|
||||
|
||||
Fetches raw bytes and metadata only.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
timeout: float = 15.0,
|
||||
headers: Optional[Mapping[str, str]] = None,
|
||||
follow_redirects: bool = True,
|
||||
):
|
||||
self.timeout = timeout
|
||||
self.headers = dict(headers) if headers else {}
|
||||
self.follow_redirects = follow_redirects
|
||||
|
||||
def fetch(
|
||||
self,
|
||||
source: str,
|
||||
*,
|
||||
metadata: Optional[Mapping[str, Any]] = None,
|
||||
) -> Content:
|
||||
with httpx.Client(
|
||||
timeout=self.timeout,
|
||||
headers=self.headers,
|
||||
follow_redirects=self.follow_redirects,
|
||||
) as client:
|
||||
response = client.get(source)
|
||||
response.raise_for_status()
|
||||
|
||||
return Content(
|
||||
raw=response.content,
|
||||
source=source,
|
||||
content_type=response.headers.get("Content-Type"),
|
||||
metadata={
|
||||
"status_code": response.status_code,
|
||||
"headers": dict(response.headers),
|
||||
},
|
||||
)
|
||||
Reference in New Issue
Block a user