feat(pdf): add PDF client, scraper, parser, and end-to-end tests
- Introduce PDF submodule with client, scraper, and generic parser - Add filesystem PDF client and test-only mock routing - Add end-to-end PDF scrape → parse tests with typed output - Mirror HTML module architecture for consistency - Expose PDF primitives via omniread public API
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
from .core import Content, ContentType
|
||||
from .html import HTMLScraper, HTMLParser
|
||||
from .pdf import FileSystemPDFClient, PDFScraper, PDFParser
|
||||
|
||||
__all__ = [
|
||||
# core
|
||||
@@ -9,4 +10,9 @@ __all__ = [
|
||||
# html
|
||||
"HTMLScraper",
|
||||
"HTMLParser",
|
||||
|
||||
# pdf
|
||||
"FileSystemPDFClient",
|
||||
"PDFScraper",
|
||||
"PDFParser",
|
||||
]
|
||||
|
||||
9
omniread/pdf/__init__.py
Normal file
9
omniread/pdf/__init__.py
Normal file
@@ -0,0 +1,9 @@
|
||||
from .client import FileSystemPDFClient
|
||||
from .scraper import PDFScraper
|
||||
from .parser import PDFParser
|
||||
|
||||
__all__ = [
|
||||
"FileSystemPDFClient",
|
||||
"PDFScraper",
|
||||
"PDFParser",
|
||||
]
|
||||
32
omniread/pdf/client.py
Normal file
32
omniread/pdf/client.py
Normal file
@@ -0,0 +1,32 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class BasePDFClient(ABC):
|
||||
"""
|
||||
Abstract client responsible for retrieving PDF bytes
|
||||
from a specific backing store (filesystem, S3, FTP, etc).
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def fetch(self, source: str) -> bytes:
|
||||
"""
|
||||
Fetch raw PDF bytes from the given source.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class FileSystemPDFClient(BasePDFClient):
|
||||
"""
|
||||
PDF client that reads from the local filesystem.
|
||||
"""
|
||||
|
||||
def fetch(self, path: Path) -> bytes:
|
||||
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"PDF not found: {path}")
|
||||
|
||||
if not path.is_file():
|
||||
raise ValueError(f"Path is not a file: {path}")
|
||||
|
||||
return path.read_bytes()
|
||||
26
omniread/pdf/parser.py
Normal file
26
omniread/pdf/parser.py
Normal file
@@ -0,0 +1,26 @@
|
||||
from typing import Generic, TypeVar
|
||||
from abc import abstractmethod
|
||||
|
||||
from omniread.core.content import ContentType
|
||||
from omniread.core.parser import BaseParser
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class PDFParser(BaseParser[T], Generic[T]):
|
||||
"""
|
||||
Base PDF parser.
|
||||
|
||||
Concrete implementations must define:
|
||||
- the output type T
|
||||
- the parsing strategy
|
||||
"""
|
||||
|
||||
supported_types = {ContentType.PDF}
|
||||
|
||||
@abstractmethod
|
||||
def parse(self) -> T:
|
||||
"""
|
||||
Parse PDF content into a structured output.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
32
omniread/pdf/scraper.py
Normal file
32
omniread/pdf/scraper.py
Normal file
@@ -0,0 +1,32 @@
|
||||
from typing import Any, Mapping, Optional
|
||||
|
||||
from omniread.core.content import Content, ContentType
|
||||
from omniread.core.scraper import BaseScraper
|
||||
from omniread.pdf.client import BasePDFClient
|
||||
|
||||
|
||||
class PDFScraper(BaseScraper):
|
||||
"""
|
||||
Scraper for PDF sources.
|
||||
|
||||
Delegates byte retrieval to a PDF client and normalizes
|
||||
output into Content.
|
||||
"""
|
||||
|
||||
def __init__(self, *, client: BasePDFClient):
|
||||
self._client = client
|
||||
|
||||
def fetch(
|
||||
self,
|
||||
source: str,
|
||||
*,
|
||||
metadata: Optional[Mapping[str, Any]] = None,
|
||||
) -> Content:
|
||||
raw = self._client.fetch(source)
|
||||
|
||||
return Content(
|
||||
raw=raw,
|
||||
source=source,
|
||||
content_type=ContentType.PDF,
|
||||
metadata=dict(metadata) if metadata else None,
|
||||
)
|
||||
Reference in New Issue
Block a user