From 664e69d8276df0259811da16b3e093ff97491db4 Mon Sep 17 00:00:00 2001 From: Vishesh 'ironeagle' Bangotra Date: Mon, 24 Aug 2026 00:53:36 +0530 Subject: [PATCH] add xlsx support: ContentType.XLSX + openpyxl-backed client/scraper/parser --- omniread/__init__.py | 17 ++++ omniread/core/content.py | 3 + omniread/xlsx/__init__.py | 27 +++++++ omniread/xlsx/client.py | 97 +++++++++++++++++++++++ omniread/xlsx/parser.py | 146 +++++++++++++++++++++++++++++++++++ omniread/xlsx/parser_base.py | 55 +++++++++++++ omniread/xlsx/scraper.py | 78 +++++++++++++++++++ pyproject.toml | 1 + tests/test_xlsx_simple.py | 118 ++++++++++++++++++++++++++++ 9 files changed, 542 insertions(+) create mode 100644 omniread/xlsx/__init__.py create mode 100644 omniread/xlsx/client.py create mode 100644 omniread/xlsx/parser.py create mode 100644 omniread/xlsx/parser_base.py create mode 100644 omniread/xlsx/scraper.py create mode 100644 tests/test_xlsx_simple.py diff --git a/omniread/__init__.py b/omniread/__init__.py index 4d26101..196bab0 100644 --- a/omniread/__init__.py +++ b/omniread/__init__.py @@ -90,6 +90,9 @@ required. - `FileSystemPDFClient`: Local filesystem PDF access. - `PDFScraper`: PDF-specific content acquisition. - `PDFParser`: Base parser for PDF binary interpretation. +- `FileSystemXlsxClient`: Local filesystem spreadsheet access. +- `XlsxScraper`: XLSX-specific content acquisition. +- `XlsxParser`: Generic string-row parser for xlsx workbooks. --- @@ -110,6 +113,13 @@ required. from .core import Content, ContentType from .html import HTMLScraper, HTMLParser from .pdf import FileSystemPDFClient, PDFScraper, PDFParser +from .xlsx import ( + BaseXlsxClient, + FileSystemXlsxClient, + XlsxParser, + XlsxParserBase, + XlsxScraper, +) __all__ = [ # core @@ -124,4 +134,11 @@ __all__ = [ "FileSystemPDFClient", "PDFScraper", "PDFParser", + + # xlsx + "BaseXlsxClient", + "FileSystemXlsxClient", + "XlsxScraper", + "XlsxParser", + "XlsxParserBase", ] diff --git a/omniread/core/content.py b/omniread/core/content.py index d929c48..65a1456 100644 --- a/omniread/core/content.py +++ b/omniread/core/content.py @@ -35,6 +35,9 @@ class ContentType(str, Enum): PDF = "application/pdf" """PDF document content.""" + XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + """Office Open XML spreadsheet (xlsx/xlsm) content.""" + JSON = "application/json" """JSON document content.""" diff --git a/omniread/xlsx/__init__.py b/omniread/xlsx/__init__.py new file mode 100644 index 0000000..a103c66 --- /dev/null +++ b/omniread/xlsx/__init__.py @@ -0,0 +1,27 @@ +""" +# Summary + +XLSX subpackage for OmniRead. + +Provides acquisition and parsing of Office Open XML spreadsheet (xlsx) +content: + +- `BaseXlsxClient`: abstract backing-store client for xlsx bytes. +- `FileSystemXlsxClient`: local filesystem implementation. +- `XlsxScraper`: wraps fetched bytes into canonical `Content`. +- `XlsxParserBase`: content-type-enforcing parser contract. +- `XlsxParser`: generic string-row parser built on openpyxl. +""" + +from .client import BaseXlsxClient, FileSystemXlsxClient +from .parser import XlsxParser +from .parser_base import XlsxParserBase +from .scraper import XlsxScraper + +__all__ = [ + "BaseXlsxClient", + "FileSystemXlsxClient", + "XlsxScraper", + "XlsxParser", + "XlsxParserBase", +] diff --git a/omniread/xlsx/client.py b/omniread/xlsx/client.py new file mode 100644 index 0000000..b7c9409 --- /dev/null +++ b/omniread/xlsx/client.py @@ -0,0 +1,97 @@ +""" +# Summary + +XLSX client abstractions for OmniRead. + +This module defines the **client layer** responsible for retrieving raw +Office Open XML spreadsheet bytes from a concrete backing store. + +Clients provide low-level access to xlsx binaries and are intentionally +decoupled from scraping and parsing logic. They do not perform validation, +interpretation, or content extraction. + +Typical backing stores include: + +- Local filesystems +- Object storage (S3, GCS, etc.) +- Network file systems +""" + +from typing import Any +from abc import ABC, abstractmethod +from pathlib import Path + + +class BaseXlsxClient(ABC): + """ + Abstract client responsible for retrieving spreadsheet bytes. + + Retrieves bytes from a specific backing store (filesystem, S3, FTP, etc.). + + Notes: + **Responsibilities:** + + - Implementations must accept a source identifier appropriate to + the backing store. + - Return the full xlsx binary payload. + - Raise retrieval-specific errors on failure. + """ + + @abstractmethod + def fetch(self, source: Any) -> bytes: + """ + Fetch raw xlsx bytes from the given source. + + Args: + source (Any): + Identifier of the spreadsheet location, such as a file path, + object storage key, or remote reference. + + Returns: + bytes: + Raw xlsx bytes. + + Raises: + Exception: + Retrieval-specific errors defined by the implementation. + """ + raise NotImplementedError + + +class FileSystemXlsxClient(BaseXlsxClient): + """ + XLSX client that reads from the local filesystem. + + Notes: + **Guarantees:** + + - This client reads spreadsheet files directly from the disk and + returns their raw binary contents. + """ + + def fetch(self, path: Path) -> bytes: + """ + Read an xlsx file from the local filesystem. + + Args: + path (Path): + Filesystem path to the spreadsheet file. + + Returns: + bytes: + Raw xlsx bytes. + + Raises: + FileNotFoundError: + If the path does not exist. + ValueError: + If the path exists but is not a file. + """ + + if not path.exists(): + raise FileNotFoundError(f"XLSX not found: {path}") + + if not path.is_file(): + raise ValueError(f"Path is not a file: {path}") + + return path.read_bytes() diff --git a/omniread/xlsx/parser.py b/omniread/xlsx/parser.py new file mode 100644 index 0000000..bb70288 --- /dev/null +++ b/omniread/xlsx/parser.py @@ -0,0 +1,146 @@ +""" +# Summary + +XLSX parser implementations for OmniRead. + +This module provides a concrete, generic parser for Office Open XML +spreadsheets. It exposes workbook sheets as lists of string rows so +downstream consumers can interpret tabular content without depending on +openpyxl directly. + +The parser is intentionally statement-agnostic: it performs no header +detection or column interpretation beyond basic cell normalization. +""" + +import datetime +from io import BytesIO +from typing import List, Optional, Union + +import openpyxl + +from omniread.core.content import Content +from .parser_base import XlsxParserBase + + +class XlsxParser(XlsxParserBase): + """ + Generic xlsx parser producing string rows from a worksheet. + + Notes: + **Responsibilities:** + + - Lazily load the workbook owned by the parser's content. + - Normalize cells (including dates and numeric values) into + deterministic string representations. + - Expose sheet discovery and row extraction helpers. + + **Constraints:** + + - Cells are rendered with ``str(value)`` after trimming; date and + datetime values are rendered in ISO format. Consumers requiring + locale-specific formatting must convert on their side. + """ + + def __init__(self, content: Content, *, data_only: bool = True, read_only: bool = True): + """ + Initialize the parser. + + Args: + content (Content): + XLSX content to parse; its type must be supported. + data_only (bool): + Passed to openpyxl: when True, formula cells yield their last + computed value instead of the formula string. + read_only (bool): + Passed to openpyxl: streaming mode for lower memory usage. + """ + super().__init__(content) + self._data_only = data_only + self._read_only = read_only + self._workbook: Optional[openpyxl.Workbook] = None + + @property + def workbook(self) -> openpyxl.Workbook: + """ + The lazily loaded workbook backing this parser's content. + """ + if self._workbook is None: + self._workbook = openpyxl.load_workbook( + BytesIO(self.content.raw), + data_only=self._data_only, + read_only=self._read_only, + ) + return self._workbook + + @property + def sheet_names(self) -> List[str]: + """ + Names of all worksheets contained in the workbook. + """ + return list(self.workbook.sheetnames) + + def parse(self) -> List[List[str]]: + """ + Parse the first worksheet into normalized string rows. + + Returns: + List[List[str]]: + Rows of the default (first) worksheet. + """ + return self.rows() + + def rows( + self, + sheet: Optional[Union[int, str]] = None, + *, + skip_empty: bool = True, + ) -> List[List[str]]: + """ + Extract normalized string rows from a worksheet. + + Args: + sheet (Optional[Union[int, str]]): + Worksheet index or title; defaults to the first worksheet. + skip_empty (bool): + When True (default), rows whose cells are all blank are omitted. + + Returns: + List[List[str]]: + Normalized rows; trailing blank cells are trimmed per row. + + Raises: + ValueError: + If the requested sheet does not exist. + """ + ws = self._resolve_sheet(sheet) + out: List[List[str]] = [] + for row in ws.iter_rows(values_only=True): + cells = [self._cell_str(v) for v in row] + while cells and not cells[-1].strip(): + cells.pop() + if skip_empty and not any(c.strip() for c in cells): + continue + out.append(cells) + return out + + def _resolve_sheet(self, sheet: Optional[Union[int, str]]): + names = self.workbook.sheetnames + if not names: + raise ValueError("Workbook contains no worksheets") + if sheet is None: + index = 0 + elif isinstance(sheet, int): + index = sheet + else: + index = names.index(sheet) if sheet in names else -1 + if not (0 <= index < len(names)): + raise ValueError(f"Worksheet not found: {sheet!r} (available: {names})") + return self.workbook[names[index]] + + @staticmethod + def _cell_str(value) -> str: + if value is None: + return "" + if isinstance(value, (datetime.datetime, datetime.date)): + return value.isoformat() + return str(value).strip() diff --git a/omniread/xlsx/parser_base.py b/omniread/xlsx/parser_base.py new file mode 100644 index 0000000..9974ef6 --- /dev/null +++ b/omniread/xlsx/parser_base.py @@ -0,0 +1,55 @@ +""" +# Summary + +XLSX parser base implementation for OmniRead. + +This module defines the **XLSX-specific parser contract**, extending the +format-agnostic `BaseParser` with constraints appropriate for Office Open +XML spreadsheet content. +""" + +from typing import Generic, TypeVar +from abc import abstractmethod + +from omniread.core.content import ContentType +from omniread.core.parser import BaseParser + +T = TypeVar("T") + + +class XlsxParserBase(BaseParser[T], Generic[T]): + """ + Base xlsx parser. + + Notes: + **Responsibilities:** + + - This class enforces xlsx content-type compatibility and provides + the extension point for implementing concrete xlsx parsing + strategies. + + **Constraints:** + + - Concrete implementations must define the output type `T` and + implement the `parse()` method. + """ + + supported_types = {ContentType.XLSX} + """ + Set of content types supported by this parser (XLSX only). + """ + + @abstractmethod + def parse(self) -> T: + """ + Parse xlsx content into a structured output. + + Returns: + T: + Parsed representation of type `T`. + + Raises: + Exception: + Parsing-specific errors as defined by the implementation. + """ + raise NotImplementedError diff --git a/omniread/xlsx/scraper.py b/omniread/xlsx/scraper.py new file mode 100644 index 0000000..00202ad --- /dev/null +++ b/omniread/xlsx/scraper.py @@ -0,0 +1,78 @@ +""" +# Summary + +XLSX scraper for OmniRead. + +This module defines the scraper responsible for acquiring raw Office Open +XML spreadsheet content from a backing store via a configured client. + +The scraper does not interpret or parse the acquired bytes; it wraps them in +the canonical `Content` model. +""" + +from typing import Any, Mapping, Optional + +from omniread.core.content import Content, ContentType +from .client import BaseXlsxClient + + +class XlsxScraper: + """ + Scraper for xlsx spreadsheet documents. + + Notes: + **Responsibilities:** + + - Fetch raw xlsx bytes via the configured client. + - Wrap the payload in a canonical `Content` instance with the + XLSX content type and source identifier. + + **Constraints:** + + - The scraper does not perform parsing or interpretation. + - Does not assume a specific storage backend. + """ + + def __init__(self, *, client: BaseXlsxClient): + """ + Initialize the XLSX scraper. + + Args: + client (BaseXlsxClient): + Client responsible for retrieving raw spreadsheet bytes. + """ + self._client = client + + def fetch( + self, + source: Any, + *, + metadata: Optional[Mapping[str, Any]] = None, + ) -> Content: + """ + Fetch an xlsx document from the given source. + + Args: + source (Any): + Identifier of the spreadsheet source as understood by the + configured client. + metadata (Optional[Mapping[str, Any]], optional): + Optional metadata to attach to the returned content. + + Returns: + Content: + A `Content` instance containing raw xlsx bytes, source + identifier, XLSX content type, and optional metadata. + + Raises: + Exception: + Retrieval-specific errors raised by the client. + """ + raw = self._client.fetch(source) + + return Content( + raw=raw, + source=source, + content_type=ContentType.XLSX, + metadata=dict(metadata) if metadata else None, + ) diff --git a/pyproject.toml b/pyproject.toml index 0219d36..fb78aff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ dependencies = [ "beautifulsoup4>=4.12.0", # "lxml>=5.0.0", "pypdf>=4.0.0", + "openpyxl>=3.1.0", ] [project.optional-dependencies] diff --git a/tests/test_xlsx_simple.py b/tests/test_xlsx_simple.py new file mode 100644 index 0000000..1390b19 --- /dev/null +++ b/tests/test_xlsx_simple.py @@ -0,0 +1,118 @@ +import datetime +from io import BytesIO + +import openpyxl +import pytest + +from omniread import ( + # core + Content, + ContentType, + + # xlsx + FileSystemXlsxClient, + XlsxParser, + XlsxScraper, +) + + +def _fixture_xlsx_bytes() -> bytes: + wb = openpyxl.Workbook() + ws = wb.active + ws.title = "Statement" + ws.append(["DETAILED STATEMENT", None, None, ""]) + ws.append([]) + ws.append(["S No", "Value Date", "Txn Date", "Remarks", "Withdrawal", "Deposit", "Balance"]) + ws.append([1, "01/06/2026", "01/06/2026", "UPI/NPCI BHIM/bhimcashback@h/B", 0.0, 10.0, 250285.24]) + row = 4 + ws.cell(row=row, column=5).value = 100.0 # ensure float rendering path + buf = BytesIO() + wb.save(buf) + return buf.getvalue() + + +@pytest.fixture +def xlsx_bytes() -> bytes: + return _fixture_xlsx_bytes() + + +@pytest.fixture +def xlsx_path(tmp_path, xlsx_bytes): + p = tmp_path / "statement.xlsx" + p.write_bytes(xlsx_bytes) + return p + + +def test_content_type_value(): + assert ContentType.XLSX.value == ( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ) + + +def test_scraper_round_trip(xlsx_path, xlsx_bytes): + scraper = XlsxScraper(client=FileSystemXlsxClient()) + content = scraper.fetch(xlsx_path) + + assert content.raw == xlsx_bytes + assert content.content_type is ContentType.XLSX + + +def test_client_missing_file_raises(tmp_path): + with pytest.raises(FileNotFoundError): + FileSystemXlsxClient().fetch(tmp_path / "nope.xlsx") + + +def test_client_directory_raises(tmp_path): + with pytest.raises(ValueError): + FileSystemXlsxClient().fetch(tmp_path) + + +def test_parser_rows_skip_empty_and_trim(xlsx_bytes): + parser = XlsxParser(Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX)) + + rows = parser.rows() + + # empty row dropped by skip_empty; metadata + header + data remain + assert len(rows) == 3 + assert rows[0] == ["DETAILED STATEMENT"] + header = rows[1] + assert header[:7] == ["S No", "Value Date", "Txn Date", "Remarks", "Withdrawal", "Deposit", "Balance"] + data = rows[2] + assert data[3] == "UPI/NPCI BHIM/bhimcashback@h/B" + assert data[6] == "250285.24" + + +def test_parser_keep_empty_rows(xlsx_bytes): + parser = XlsxParser(Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX)) + + rows = parser.rows(skip_empty=False) + + assert len(rows) == 4 # includes blank second row and padded metadata row + + +def test_parser_sheet_selection_and_names(xlsx_bytes): + parser = XlsxParser(Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX)) + + assert parser.sheet_names == ["Statement"] + assert parser.rows(sheet="Statement") == parser.rows(sheet=0) + assert parser.parse() == parser.rows() + + with pytest.raises(ValueError, match="not found"): + parser.rows(sheet="Missing") + + +def test_parser_date_cell_isoformat(): + wb = openpyxl.Workbook() + ws = wb.active + ws.append([datetime.date(2026, 6, 1), "x"]) + buf = BytesIO() + wb.save(buf) + + parser = XlsxParser(Content(raw=buf.getvalue(), source="mem", content_type=ContentType.XLSX)) + # openpyxl surfaces date cells as datetime; ISO rendering keeps determinism + assert parser.parse()[0][0] == "2026-06-01T00:00:00" + + +def test_parser_rejects_non_xlsx_content(): + with pytest.raises(ValueError, match="does not support"): + XlsxParser(Content(raw=b"", source="mem", content_type=ContentType.HTML))