Compare commits
2 Commits
lib-standa
...
ed388fe59f
| Author | SHA1 | Date | |
|---|---|---|---|
| ed388fe59f | |||
| 664e69d827 |
@@ -90,6 +90,9 @@ required.
|
|||||||
- `FileSystemPDFClient`: Local filesystem PDF access.
|
- `FileSystemPDFClient`: Local filesystem PDF access.
|
||||||
- `PDFScraper`: PDF-specific content acquisition.
|
- `PDFScraper`: PDF-specific content acquisition.
|
||||||
- `PDFParser`: Base parser for PDF binary interpretation.
|
- `PDFParser`: Base parser for PDF binary interpretation.
|
||||||
|
- `FileSystemXlsxClient`: Local filesystem spreadsheet access.
|
||||||
|
- `XlsxScraper`: XLSX-specific content acquisition.
|
||||||
|
- `XlsxParser`: Generic string-row parser for xlsx workbooks.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -108,8 +111,22 @@ required.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
from .core import Content, ContentType
|
from .core import Content, ContentType
|
||||||
|
from .csv import (
|
||||||
|
BaseCsvClient,
|
||||||
|
CsvParser,
|
||||||
|
CsvParserBase,
|
||||||
|
CsvScraper,
|
||||||
|
FileSystemCsvClient,
|
||||||
|
)
|
||||||
from .html import HTMLScraper, HTMLParser
|
from .html import HTMLScraper, HTMLParser
|
||||||
from .pdf import FileSystemPDFClient, PDFScraper, PDFParser
|
from .pdf import FileSystemPDFClient, PDFScraper, PDFParser
|
||||||
|
from .xlsx import (
|
||||||
|
BaseXlsxClient,
|
||||||
|
FileSystemXlsxClient,
|
||||||
|
XlsxParser,
|
||||||
|
XlsxParserBase,
|
||||||
|
XlsxScraper,
|
||||||
|
)
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
# core
|
# core
|
||||||
@@ -124,4 +141,18 @@ __all__ = [
|
|||||||
"FileSystemPDFClient",
|
"FileSystemPDFClient",
|
||||||
"PDFScraper",
|
"PDFScraper",
|
||||||
"PDFParser",
|
"PDFParser",
|
||||||
|
|
||||||
|
# csv
|
||||||
|
"BaseCsvClient",
|
||||||
|
"FileSystemCsvClient",
|
||||||
|
"CsvScraper",
|
||||||
|
"CsvParser",
|
||||||
|
"CsvParserBase",
|
||||||
|
|
||||||
|
# xlsx
|
||||||
|
"BaseXlsxClient",
|
||||||
|
"FileSystemXlsxClient",
|
||||||
|
"XlsxScraper",
|
||||||
|
"XlsxParser",
|
||||||
|
"XlsxParserBase",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -35,6 +35,12 @@ class ContentType(str, Enum):
|
|||||||
PDF = "application/pdf"
|
PDF = "application/pdf"
|
||||||
"""PDF document content."""
|
"""PDF document content."""
|
||||||
|
|
||||||
|
XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||||||
|
"""Office Open XML spreadsheet (xlsx/xlsm) content."""
|
||||||
|
|
||||||
|
CSV = "text/csv"
|
||||||
|
"""Comma-separated-value document content."""
|
||||||
|
|
||||||
JSON = "application/json"
|
JSON = "application/json"
|
||||||
"""JSON document content."""
|
"""JSON document content."""
|
||||||
|
|
||||||
|
|||||||
26
omniread/csv/__init__.py
Normal file
26
omniread/csv/__init__.py
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
|
CSV subpackage for OmniRead.
|
||||||
|
|
||||||
|
Provides acquisition and parsing of comma-separated-value content:
|
||||||
|
|
||||||
|
- `BaseCsvClient`: abstract backing-store client for csv bytes.
|
||||||
|
- `FileSystemCsvClient`: local filesystem implementation.
|
||||||
|
- `CsvScraper`: wraps fetched bytes into canonical `Content`.
|
||||||
|
- `CsvParserBase`: content-type-enforcing parser contract.
|
||||||
|
- `CsvParser`: generic string-row parser built on the standard csv module.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from .client import BaseCsvClient, FileSystemCsvClient
|
||||||
|
from .parser import CsvParser
|
||||||
|
from .parser_base import CsvParserBase
|
||||||
|
from .scraper import CsvScraper
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"BaseCsvClient",
|
||||||
|
"FileSystemCsvClient",
|
||||||
|
"CsvScraper",
|
||||||
|
"CsvParser",
|
||||||
|
"CsvParserBase",
|
||||||
|
]
|
||||||
97
omniread/csv/client.py
Normal file
97
omniread/csv/client.py
Normal file
@@ -0,0 +1,97 @@
|
|||||||
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
|
CSV client abstractions for OmniRead.
|
||||||
|
|
||||||
|
This module defines the **client layer** responsible for retrieving raw
|
||||||
|
comma-separated-value document bytes from a concrete backing store.
|
||||||
|
|
||||||
|
Clients provide low-level access to csv binaries and are intentionally
|
||||||
|
decoupled from scraping and parsing logic. They do not perform validation,
|
||||||
|
interpretation, or content extraction.
|
||||||
|
|
||||||
|
Typical backing stores include:
|
||||||
|
|
||||||
|
- Local filesystems
|
||||||
|
- Object storage (S3, GCS, etc.)
|
||||||
|
- Network file systems
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
from abc import ABC, abstractmethod
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
class BaseCsvClient(ABC):
|
||||||
|
"""
|
||||||
|
Abstract client responsible for retrieving csv bytes.
|
||||||
|
|
||||||
|
Retrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
**Responsibilities:**
|
||||||
|
|
||||||
|
- Implementations must accept a source identifier appropriate to
|
||||||
|
the backing store.
|
||||||
|
- Return the full csv binary payload.
|
||||||
|
- Raise retrieval-specific errors on failure.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def fetch(self, source: Any) -> bytes:
|
||||||
|
"""
|
||||||
|
Fetch raw csv bytes from the given source.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
source (Any):
|
||||||
|
Identifier of the csv location, such as a file path,
|
||||||
|
object storage key, or remote reference.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
bytes:
|
||||||
|
Raw csv bytes.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
Exception:
|
||||||
|
Retrieval-specific errors defined by the implementation.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
|
||||||
|
class FileSystemCsvClient(BaseCsvClient):
|
||||||
|
"""
|
||||||
|
CSV client that reads from the local filesystem.
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
**Guarantees:**
|
||||||
|
|
||||||
|
- This client reads csv files directly from the disk and
|
||||||
|
returns their raw binary contents.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def fetch(self, path: Path) -> bytes:
|
||||||
|
"""
|
||||||
|
Read a csv file from the local filesystem.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
path (Path):
|
||||||
|
Filesystem path to the csv file.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
bytes:
|
||||||
|
Raw csv bytes.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
FileNotFoundError:
|
||||||
|
If the path does not exist.
|
||||||
|
ValueError:
|
||||||
|
If the path exists but is not a file.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if not path.exists():
|
||||||
|
raise FileNotFoundError(f"csv not found: {path}")
|
||||||
|
|
||||||
|
if not path.is_file():
|
||||||
|
raise ValueError(f"Path is not a file: {path}")
|
||||||
|
|
||||||
|
return path.read_bytes()
|
||||||
102
omniread/csv/parser.py
Normal file
102
omniread/csv/parser.py
Normal file
@@ -0,0 +1,102 @@
|
|||||||
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
|
CSV parser implementations for OmniRead.
|
||||||
|
|
||||||
|
This module provides a concrete, generic parser for comma-separated-value
|
||||||
|
documents. It exposes records as lists of string cells so downstream
|
||||||
|
consumers can interpret tabular content without depending on the ``csv``
|
||||||
|
module directly.
|
||||||
|
|
||||||
|
The parser is intentionally statement-agnostic: it performs no header
|
||||||
|
detection or column interpretation beyond basic cell normalization and
|
||||||
|
delimiter detection.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from io import StringIO
|
||||||
|
from csv import Sniffer, reader
|
||||||
|
from typing import List
|
||||||
|
|
||||||
|
from omniread.core.content import Content
|
||||||
|
from .parser_base import CsvParserBase
|
||||||
|
|
||||||
|
|
||||||
|
class CsvParser(CsvParserBase):
|
||||||
|
"""
|
||||||
|
Generic csv parser producing string rows from the document.
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
**Responsibilities:**
|
||||||
|
|
||||||
|
- Decode the payload (UTF-8 with BOM support, Latin-1 fallback).
|
||||||
|
- Detect the delimiter from a leading sample (`,` `;` tab `|`),
|
||||||
|
defaulting to `,`.
|
||||||
|
- Normalize cells into deterministic stripped string values.
|
||||||
|
- Expose row extraction helpers mirroring `XlsxParser.rows`.
|
||||||
|
|
||||||
|
**Constraints:**
|
||||||
|
|
||||||
|
- All values are strings; consumers requiring typed values must
|
||||||
|
convert on their side.
|
||||||
|
- Quoted fields containing delimiters/newlines are handled by
|
||||||
|
the standard ``csv`` module.
|
||||||
|
"""
|
||||||
|
|
||||||
|
_DELIMITERS = ",;\t|"
|
||||||
|
|
||||||
|
def __init__(self, content: Content):
|
||||||
|
"""
|
||||||
|
Initialize the parser.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
content (Content):
|
||||||
|
CSV content to parse; its type must be supported.
|
||||||
|
"""
|
||||||
|
super().__init__(content)
|
||||||
|
|
||||||
|
def parse(self) -> List[List[str]]:
|
||||||
|
"""
|
||||||
|
Parse the document into normalized string rows.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List[List[str]]:
|
||||||
|
Rows of the document.
|
||||||
|
"""
|
||||||
|
return self.rows()
|
||||||
|
|
||||||
|
def rows(self, *, skip_empty: bool = True) -> List[List[str]]:
|
||||||
|
"""
|
||||||
|
Extract normalized string rows from the document.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
skip_empty (bool):
|
||||||
|
When True (default), rows whose cells are all blank are
|
||||||
|
omitted.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List[List[str]]:
|
||||||
|
Normalized rows; trailing blank cells are trimmed per row.
|
||||||
|
"""
|
||||||
|
text = self._decode(self.content.raw)
|
||||||
|
dialect_sample = text[:4096]
|
||||||
|
try:
|
||||||
|
dialect = Sniffer().sniff(dialect_sample, delimiters=self._DELIMITERS)
|
||||||
|
delimiter = dialect.delimiter
|
||||||
|
except Exception:
|
||||||
|
delimiter = ","
|
||||||
|
out: List[List[str]] = []
|
||||||
|
for row in reader(StringIO(text), delimiter=delimiter):
|
||||||
|
cells = [c.strip() for c in row]
|
||||||
|
while cells and not cells[-1]:
|
||||||
|
cells.pop()
|
||||||
|
if skip_empty and not any(cells):
|
||||||
|
continue
|
||||||
|
out.append(cells)
|
||||||
|
return out
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _decode(raw: bytes) -> str:
|
||||||
|
try:
|
||||||
|
return raw.decode("utf-8-sig")
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
return raw.decode("latin-1")
|
||||||
55
omniread/csv/parser_base.py
Normal file
55
omniread/csv/parser_base.py
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
|
CSV parser base implementation for OmniRead.
|
||||||
|
|
||||||
|
This module defines the **CSV-specific parser contract**, extending the
|
||||||
|
format-agnostic `BaseParser` with constraints appropriate for
|
||||||
|
comma-separated-value documents.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Generic, TypeVar
|
||||||
|
from abc import abstractmethod
|
||||||
|
|
||||||
|
from omniread.core.content import ContentType
|
||||||
|
from omniread.core.parser import BaseParser
|
||||||
|
|
||||||
|
T = TypeVar("T")
|
||||||
|
|
||||||
|
|
||||||
|
class CsvParserBase(BaseParser[T], Generic[T]):
|
||||||
|
"""
|
||||||
|
Base csv parser.
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
**Responsibilities:**
|
||||||
|
|
||||||
|
- This class enforces csv content-type compatibility and provides
|
||||||
|
the extension point for implementing concrete csv parsing
|
||||||
|
strategies.
|
||||||
|
|
||||||
|
**Constraints:**
|
||||||
|
|
||||||
|
- Concrete implementations must define the output type `T` and
|
||||||
|
implement the `parse()` method.
|
||||||
|
"""
|
||||||
|
|
||||||
|
supported_types = {ContentType.CSV}
|
||||||
|
"""
|
||||||
|
Set of content types supported by this parser (CSV only).
|
||||||
|
"""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def parse(self) -> T:
|
||||||
|
"""
|
||||||
|
Parse csv content into a structured output.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
T:
|
||||||
|
Parsed representation of type `T`.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
Exception:
|
||||||
|
Parsing-specific errors as defined by the implementation.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
79
omniread/csv/scraper.py
Normal file
79
omniread/csv/scraper.py
Normal file
@@ -0,0 +1,79 @@
|
|||||||
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
|
CSV scraper for OmniRead.
|
||||||
|
|
||||||
|
This module defines the scraper responsible for acquiring raw
|
||||||
|
comma-separated-value document content from a backing store via a
|
||||||
|
configured client.
|
||||||
|
|
||||||
|
The scraper does not interpret or parse the acquired bytes; it wraps them in
|
||||||
|
the canonical `Content` model.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Any, Mapping, Optional
|
||||||
|
|
||||||
|
from omniread.core.content import Content, ContentType
|
||||||
|
from .client import BaseCsvClient
|
||||||
|
|
||||||
|
|
||||||
|
class CsvScraper:
|
||||||
|
"""
|
||||||
|
Scraper for csv documents.
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
**Responsibilities:**
|
||||||
|
|
||||||
|
- Fetch raw csv bytes via the configured client.
|
||||||
|
- Wrap the payload in a canonical `Content` instance with the
|
||||||
|
CSV content type and source identifier.
|
||||||
|
|
||||||
|
**Constraints:**
|
||||||
|
|
||||||
|
- The scraper does not perform parsing or interpretation.
|
||||||
|
- Does not assume a specific storage backend.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, *, client: BaseCsvClient):
|
||||||
|
"""
|
||||||
|
Initialize the CSV scraper.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
client (BaseCsvClient):
|
||||||
|
Client responsible for retrieving raw csv bytes.
|
||||||
|
"""
|
||||||
|
self._client = client
|
||||||
|
|
||||||
|
def fetch(
|
||||||
|
self,
|
||||||
|
source: Any,
|
||||||
|
*,
|
||||||
|
metadata: Optional[Mapping[str, Any]] = None,
|
||||||
|
) -> Content:
|
||||||
|
"""
|
||||||
|
Fetch a csv document from the given source.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
source (Any):
|
||||||
|
Identifier of the csv source as understood by the
|
||||||
|
configured client.
|
||||||
|
metadata (Optional[Mapping[str, Any]], optional):
|
||||||
|
Optional metadata to attach to the returned content.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Content:
|
||||||
|
A `Content` instance containing raw csv bytes, source
|
||||||
|
identifier, CSV content type, and optional metadata.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
Exception:
|
||||||
|
Retrieval-specific errors raised by the client.
|
||||||
|
"""
|
||||||
|
raw = self._client.fetch(source)
|
||||||
|
|
||||||
|
return Content(
|
||||||
|
raw=raw,
|
||||||
|
source=source,
|
||||||
|
content_type=ContentType.CSV,
|
||||||
|
metadata=dict(metadata) if metadata else None,
|
||||||
|
)
|
||||||
27
omniread/xlsx/__init__.py
Normal file
27
omniread/xlsx/__init__.py
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
|
XLSX subpackage for OmniRead.
|
||||||
|
|
||||||
|
Provides acquisition and parsing of Office Open XML spreadsheet (xlsx)
|
||||||
|
content:
|
||||||
|
|
||||||
|
- `BaseXlsxClient`: abstract backing-store client for xlsx bytes.
|
||||||
|
- `FileSystemXlsxClient`: local filesystem implementation.
|
||||||
|
- `XlsxScraper`: wraps fetched bytes into canonical `Content`.
|
||||||
|
- `XlsxParserBase`: content-type-enforcing parser contract.
|
||||||
|
- `XlsxParser`: generic string-row parser built on openpyxl.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from .client import BaseXlsxClient, FileSystemXlsxClient
|
||||||
|
from .parser import XlsxParser
|
||||||
|
from .parser_base import XlsxParserBase
|
||||||
|
from .scraper import XlsxScraper
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"BaseXlsxClient",
|
||||||
|
"FileSystemXlsxClient",
|
||||||
|
"XlsxScraper",
|
||||||
|
"XlsxParser",
|
||||||
|
"XlsxParserBase",
|
||||||
|
]
|
||||||
97
omniread/xlsx/client.py
Normal file
97
omniread/xlsx/client.py
Normal file
@@ -0,0 +1,97 @@
|
|||||||
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
|
XLSX client abstractions for OmniRead.
|
||||||
|
|
||||||
|
This module defines the **client layer** responsible for retrieving raw
|
||||||
|
Office Open XML spreadsheet bytes from a concrete backing store.
|
||||||
|
|
||||||
|
Clients provide low-level access to xlsx binaries and are intentionally
|
||||||
|
decoupled from scraping and parsing logic. They do not perform validation,
|
||||||
|
interpretation, or content extraction.
|
||||||
|
|
||||||
|
Typical backing stores include:
|
||||||
|
|
||||||
|
- Local filesystems
|
||||||
|
- Object storage (S3, GCS, etc.)
|
||||||
|
- Network file systems
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
from abc import ABC, abstractmethod
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
class BaseXlsxClient(ABC):
|
||||||
|
"""
|
||||||
|
Abstract client responsible for retrieving spreadsheet bytes.
|
||||||
|
|
||||||
|
Retrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
**Responsibilities:**
|
||||||
|
|
||||||
|
- Implementations must accept a source identifier appropriate to
|
||||||
|
the backing store.
|
||||||
|
- Return the full xlsx binary payload.
|
||||||
|
- Raise retrieval-specific errors on failure.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def fetch(self, source: Any) -> bytes:
|
||||||
|
"""
|
||||||
|
Fetch raw xlsx bytes from the given source.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
source (Any):
|
||||||
|
Identifier of the spreadsheet location, such as a file path,
|
||||||
|
object storage key, or remote reference.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
bytes:
|
||||||
|
Raw xlsx bytes.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
Exception:
|
||||||
|
Retrieval-specific errors defined by the implementation.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
|
||||||
|
class FileSystemXlsxClient(BaseXlsxClient):
|
||||||
|
"""
|
||||||
|
XLSX client that reads from the local filesystem.
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
**Guarantees:**
|
||||||
|
|
||||||
|
- This client reads spreadsheet files directly from the disk and
|
||||||
|
returns their raw binary contents.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def fetch(self, path: Path) -> bytes:
|
||||||
|
"""
|
||||||
|
Read an xlsx file from the local filesystem.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
path (Path):
|
||||||
|
Filesystem path to the spreadsheet file.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
bytes:
|
||||||
|
Raw xlsx bytes.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
FileNotFoundError:
|
||||||
|
If the path does not exist.
|
||||||
|
ValueError:
|
||||||
|
If the path exists but is not a file.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if not path.exists():
|
||||||
|
raise FileNotFoundError(f"XLSX not found: {path}")
|
||||||
|
|
||||||
|
if not path.is_file():
|
||||||
|
raise ValueError(f"Path is not a file: {path}")
|
||||||
|
|
||||||
|
return path.read_bytes()
|
||||||
146
omniread/xlsx/parser.py
Normal file
146
omniread/xlsx/parser.py
Normal file
@@ -0,0 +1,146 @@
|
|||||||
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
|
XLSX parser implementations for OmniRead.
|
||||||
|
|
||||||
|
This module provides a concrete, generic parser for Office Open XML
|
||||||
|
spreadsheets. It exposes workbook sheets as lists of string rows so
|
||||||
|
downstream consumers can interpret tabular content without depending on
|
||||||
|
openpyxl directly.
|
||||||
|
|
||||||
|
The parser is intentionally statement-agnostic: it performs no header
|
||||||
|
detection or column interpretation beyond basic cell normalization.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import datetime
|
||||||
|
from io import BytesIO
|
||||||
|
from typing import List, Optional, Union
|
||||||
|
|
||||||
|
import openpyxl
|
||||||
|
|
||||||
|
from omniread.core.content import Content
|
||||||
|
from .parser_base import XlsxParserBase
|
||||||
|
|
||||||
|
|
||||||
|
class XlsxParser(XlsxParserBase):
|
||||||
|
"""
|
||||||
|
Generic xlsx parser producing string rows from a worksheet.
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
**Responsibilities:**
|
||||||
|
|
||||||
|
- Lazily load the workbook owned by the parser's content.
|
||||||
|
- Normalize cells (including dates and numeric values) into
|
||||||
|
deterministic string representations.
|
||||||
|
- Expose sheet discovery and row extraction helpers.
|
||||||
|
|
||||||
|
**Constraints:**
|
||||||
|
|
||||||
|
- Cells are rendered with ``str(value)`` after trimming; date and
|
||||||
|
datetime values are rendered in ISO format. Consumers requiring
|
||||||
|
locale-specific formatting must convert on their side.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, content: Content, *, data_only: bool = True, read_only: bool = True):
|
||||||
|
"""
|
||||||
|
Initialize the parser.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
content (Content):
|
||||||
|
XLSX content to parse; its type must be supported.
|
||||||
|
data_only (bool):
|
||||||
|
Passed to openpyxl: when True, formula cells yield their last
|
||||||
|
computed value instead of the formula string.
|
||||||
|
read_only (bool):
|
||||||
|
Passed to openpyxl: streaming mode for lower memory usage.
|
||||||
|
"""
|
||||||
|
super().__init__(content)
|
||||||
|
self._data_only = data_only
|
||||||
|
self._read_only = read_only
|
||||||
|
self._workbook: Optional[openpyxl.Workbook] = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def workbook(self) -> openpyxl.Workbook:
|
||||||
|
"""
|
||||||
|
The lazily loaded workbook backing this parser's content.
|
||||||
|
"""
|
||||||
|
if self._workbook is None:
|
||||||
|
self._workbook = openpyxl.load_workbook(
|
||||||
|
BytesIO(self.content.raw),
|
||||||
|
data_only=self._data_only,
|
||||||
|
read_only=self._read_only,
|
||||||
|
)
|
||||||
|
return self._workbook
|
||||||
|
|
||||||
|
@property
|
||||||
|
def sheet_names(self) -> List[str]:
|
||||||
|
"""
|
||||||
|
Names of all worksheets contained in the workbook.
|
||||||
|
"""
|
||||||
|
return list(self.workbook.sheetnames)
|
||||||
|
|
||||||
|
def parse(self) -> List[List[str]]:
|
||||||
|
"""
|
||||||
|
Parse the first worksheet into normalized string rows.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List[List[str]]:
|
||||||
|
Rows of the default (first) worksheet.
|
||||||
|
"""
|
||||||
|
return self.rows()
|
||||||
|
|
||||||
|
def rows(
|
||||||
|
self,
|
||||||
|
sheet: Optional[Union[int, str]] = None,
|
||||||
|
*,
|
||||||
|
skip_empty: bool = True,
|
||||||
|
) -> List[List[str]]:
|
||||||
|
"""
|
||||||
|
Extract normalized string rows from a worksheet.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
sheet (Optional[Union[int, str]]):
|
||||||
|
Worksheet index or title; defaults to the first worksheet.
|
||||||
|
skip_empty (bool):
|
||||||
|
When True (default), rows whose cells are all blank are omitted.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List[List[str]]:
|
||||||
|
Normalized rows; trailing blank cells are trimmed per row.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ValueError:
|
||||||
|
If the requested sheet does not exist.
|
||||||
|
"""
|
||||||
|
ws = self._resolve_sheet(sheet)
|
||||||
|
out: List[List[str]] = []
|
||||||
|
for row in ws.iter_rows(values_only=True):
|
||||||
|
cells = [self._cell_str(v) for v in row]
|
||||||
|
while cells and not cells[-1].strip():
|
||||||
|
cells.pop()
|
||||||
|
if skip_empty and not any(c.strip() for c in cells):
|
||||||
|
continue
|
||||||
|
out.append(cells)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _resolve_sheet(self, sheet: Optional[Union[int, str]]):
|
||||||
|
names = self.workbook.sheetnames
|
||||||
|
if not names:
|
||||||
|
raise ValueError("Workbook contains no worksheets")
|
||||||
|
if sheet is None:
|
||||||
|
index = 0
|
||||||
|
elif isinstance(sheet, int):
|
||||||
|
index = sheet
|
||||||
|
else:
|
||||||
|
index = names.index(sheet) if sheet in names else -1
|
||||||
|
if not (0 <= index < len(names)):
|
||||||
|
raise ValueError(f"Worksheet not found: {sheet!r} (available: {names})")
|
||||||
|
return self.workbook[names[index]]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _cell_str(value) -> str:
|
||||||
|
if value is None:
|
||||||
|
return ""
|
||||||
|
if isinstance(value, (datetime.datetime, datetime.date)):
|
||||||
|
return value.isoformat()
|
||||||
|
return str(value).strip()
|
||||||
55
omniread/xlsx/parser_base.py
Normal file
55
omniread/xlsx/parser_base.py
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
|
XLSX parser base implementation for OmniRead.
|
||||||
|
|
||||||
|
This module defines the **XLSX-specific parser contract**, extending the
|
||||||
|
format-agnostic `BaseParser` with constraints appropriate for Office Open
|
||||||
|
XML spreadsheet content.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Generic, TypeVar
|
||||||
|
from abc import abstractmethod
|
||||||
|
|
||||||
|
from omniread.core.content import ContentType
|
||||||
|
from omniread.core.parser import BaseParser
|
||||||
|
|
||||||
|
T = TypeVar("T")
|
||||||
|
|
||||||
|
|
||||||
|
class XlsxParserBase(BaseParser[T], Generic[T]):
|
||||||
|
"""
|
||||||
|
Base xlsx parser.
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
**Responsibilities:**
|
||||||
|
|
||||||
|
- This class enforces xlsx content-type compatibility and provides
|
||||||
|
the extension point for implementing concrete xlsx parsing
|
||||||
|
strategies.
|
||||||
|
|
||||||
|
**Constraints:**
|
||||||
|
|
||||||
|
- Concrete implementations must define the output type `T` and
|
||||||
|
implement the `parse()` method.
|
||||||
|
"""
|
||||||
|
|
||||||
|
supported_types = {ContentType.XLSX}
|
||||||
|
"""
|
||||||
|
Set of content types supported by this parser (XLSX only).
|
||||||
|
"""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def parse(self) -> T:
|
||||||
|
"""
|
||||||
|
Parse xlsx content into a structured output.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
T:
|
||||||
|
Parsed representation of type `T`.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
Exception:
|
||||||
|
Parsing-specific errors as defined by the implementation.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
78
omniread/xlsx/scraper.py
Normal file
78
omniread/xlsx/scraper.py
Normal file
@@ -0,0 +1,78 @@
|
|||||||
|
"""
|
||||||
|
# Summary
|
||||||
|
|
||||||
|
XLSX scraper for OmniRead.
|
||||||
|
|
||||||
|
This module defines the scraper responsible for acquiring raw Office Open
|
||||||
|
XML spreadsheet content from a backing store via a configured client.
|
||||||
|
|
||||||
|
The scraper does not interpret or parse the acquired bytes; it wraps them in
|
||||||
|
the canonical `Content` model.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Any, Mapping, Optional
|
||||||
|
|
||||||
|
from omniread.core.content import Content, ContentType
|
||||||
|
from .client import BaseXlsxClient
|
||||||
|
|
||||||
|
|
||||||
|
class XlsxScraper:
|
||||||
|
"""
|
||||||
|
Scraper for xlsx spreadsheet documents.
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
**Responsibilities:**
|
||||||
|
|
||||||
|
- Fetch raw xlsx bytes via the configured client.
|
||||||
|
- Wrap the payload in a canonical `Content` instance with the
|
||||||
|
XLSX content type and source identifier.
|
||||||
|
|
||||||
|
**Constraints:**
|
||||||
|
|
||||||
|
- The scraper does not perform parsing or interpretation.
|
||||||
|
- Does not assume a specific storage backend.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, *, client: BaseXlsxClient):
|
||||||
|
"""
|
||||||
|
Initialize the XLSX scraper.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
client (BaseXlsxClient):
|
||||||
|
Client responsible for retrieving raw spreadsheet bytes.
|
||||||
|
"""
|
||||||
|
self._client = client
|
||||||
|
|
||||||
|
def fetch(
|
||||||
|
self,
|
||||||
|
source: Any,
|
||||||
|
*,
|
||||||
|
metadata: Optional[Mapping[str, Any]] = None,
|
||||||
|
) -> Content:
|
||||||
|
"""
|
||||||
|
Fetch an xlsx document from the given source.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
source (Any):
|
||||||
|
Identifier of the spreadsheet source as understood by the
|
||||||
|
configured client.
|
||||||
|
metadata (Optional[Mapping[str, Any]], optional):
|
||||||
|
Optional metadata to attach to the returned content.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Content:
|
||||||
|
A `Content` instance containing raw xlsx bytes, source
|
||||||
|
identifier, XLSX content type, and optional metadata.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
Exception:
|
||||||
|
Retrieval-specific errors raised by the client.
|
||||||
|
"""
|
||||||
|
raw = self._client.fetch(source)
|
||||||
|
|
||||||
|
return Content(
|
||||||
|
raw=raw,
|
||||||
|
source=source,
|
||||||
|
content_type=ContentType.XLSX,
|
||||||
|
metadata=dict(metadata) if metadata else None,
|
||||||
|
)
|
||||||
@@ -48,6 +48,7 @@ dependencies = [
|
|||||||
"beautifulsoup4>=4.12.0",
|
"beautifulsoup4>=4.12.0",
|
||||||
# "lxml>=5.0.0",
|
# "lxml>=5.0.0",
|
||||||
"pypdf>=4.0.0",
|
"pypdf>=4.0.0",
|
||||||
|
"openpyxl>=3.1.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
|
|||||||
118
tests/test_xlsx_simple.py
Normal file
118
tests/test_xlsx_simple.py
Normal file
@@ -0,0 +1,118 @@
|
|||||||
|
import datetime
|
||||||
|
from io import BytesIO
|
||||||
|
|
||||||
|
import openpyxl
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from omniread import (
|
||||||
|
# core
|
||||||
|
Content,
|
||||||
|
ContentType,
|
||||||
|
|
||||||
|
# xlsx
|
||||||
|
FileSystemXlsxClient,
|
||||||
|
XlsxParser,
|
||||||
|
XlsxScraper,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _fixture_xlsx_bytes() -> bytes:
|
||||||
|
wb = openpyxl.Workbook()
|
||||||
|
ws = wb.active
|
||||||
|
ws.title = "Statement"
|
||||||
|
ws.append(["DETAILED STATEMENT", None, None, ""])
|
||||||
|
ws.append([])
|
||||||
|
ws.append(["S No", "Value Date", "Txn Date", "Remarks", "Withdrawal", "Deposit", "Balance"])
|
||||||
|
ws.append([1, "01/06/2026", "01/06/2026", "UPI/NPCI BHIM/bhimcashback@h/B", 0.0, 10.0, 250285.24])
|
||||||
|
row = 4
|
||||||
|
ws.cell(row=row, column=5).value = 100.0 # ensure float rendering path
|
||||||
|
buf = BytesIO()
|
||||||
|
wb.save(buf)
|
||||||
|
return buf.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def xlsx_bytes() -> bytes:
|
||||||
|
return _fixture_xlsx_bytes()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def xlsx_path(tmp_path, xlsx_bytes):
|
||||||
|
p = tmp_path / "statement.xlsx"
|
||||||
|
p.write_bytes(xlsx_bytes)
|
||||||
|
return p
|
||||||
|
|
||||||
|
|
||||||
|
def test_content_type_value():
|
||||||
|
assert ContentType.XLSX.value == (
|
||||||
|
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_scraper_round_trip(xlsx_path, xlsx_bytes):
|
||||||
|
scraper = XlsxScraper(client=FileSystemXlsxClient())
|
||||||
|
content = scraper.fetch(xlsx_path)
|
||||||
|
|
||||||
|
assert content.raw == xlsx_bytes
|
||||||
|
assert content.content_type is ContentType.XLSX
|
||||||
|
|
||||||
|
|
||||||
|
def test_client_missing_file_raises(tmp_path):
|
||||||
|
with pytest.raises(FileNotFoundError):
|
||||||
|
FileSystemXlsxClient().fetch(tmp_path / "nope.xlsx")
|
||||||
|
|
||||||
|
|
||||||
|
def test_client_directory_raises(tmp_path):
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
FileSystemXlsxClient().fetch(tmp_path)
|
||||||
|
|
||||||
|
|
||||||
|
def test_parser_rows_skip_empty_and_trim(xlsx_bytes):
|
||||||
|
parser = XlsxParser(Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX))
|
||||||
|
|
||||||
|
rows = parser.rows()
|
||||||
|
|
||||||
|
# empty row dropped by skip_empty; metadata + header + data remain
|
||||||
|
assert len(rows) == 3
|
||||||
|
assert rows[0] == ["DETAILED STATEMENT"]
|
||||||
|
header = rows[1]
|
||||||
|
assert header[:7] == ["S No", "Value Date", "Txn Date", "Remarks", "Withdrawal", "Deposit", "Balance"]
|
||||||
|
data = rows[2]
|
||||||
|
assert data[3] == "UPI/NPCI BHIM/bhimcashback@h/B"
|
||||||
|
assert data[6] == "250285.24"
|
||||||
|
|
||||||
|
|
||||||
|
def test_parser_keep_empty_rows(xlsx_bytes):
|
||||||
|
parser = XlsxParser(Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX))
|
||||||
|
|
||||||
|
rows = parser.rows(skip_empty=False)
|
||||||
|
|
||||||
|
assert len(rows) == 4 # includes blank second row and padded metadata row
|
||||||
|
|
||||||
|
|
||||||
|
def test_parser_sheet_selection_and_names(xlsx_bytes):
|
||||||
|
parser = XlsxParser(Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX))
|
||||||
|
|
||||||
|
assert parser.sheet_names == ["Statement"]
|
||||||
|
assert parser.rows(sheet="Statement") == parser.rows(sheet=0)
|
||||||
|
assert parser.parse() == parser.rows()
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="not found"):
|
||||||
|
parser.rows(sheet="Missing")
|
||||||
|
|
||||||
|
|
||||||
|
def test_parser_date_cell_isoformat():
|
||||||
|
wb = openpyxl.Workbook()
|
||||||
|
ws = wb.active
|
||||||
|
ws.append([datetime.date(2026, 6, 1), "x"])
|
||||||
|
buf = BytesIO()
|
||||||
|
wb.save(buf)
|
||||||
|
|
||||||
|
parser = XlsxParser(Content(raw=buf.getvalue(), source="mem", content_type=ContentType.XLSX))
|
||||||
|
# openpyxl surfaces date cells as datetime; ISO rendering keeps determinism
|
||||||
|
assert parser.parse()[0][0] == "2026-06-01T00:00:00"
|
||||||
|
|
||||||
|
|
||||||
|
def test_parser_rejects_non_xlsx_content():
|
||||||
|
with pytest.raises(ValueError, match="does not support"):
|
||||||
|
XlsxParser(Content(raw=b"<html></html>", source="mem", content_type=ContentType.HTML))
|
||||||
Reference in New Issue
Block a user