xlsx-and-csv-read (#3)

Reviewed-on: #3
Co-authored-by: Vishesh 'ironeagle' Bangotra <aetoskia@gmail.com>
Co-committed-by: Vishesh 'ironeagle' Bangotra <aetoskia@gmail.com>
This commit is contained in:
2026-08-24 20:19:33 +00:00
parent de7d04eb1a
commit 04d8e069d4
14 changed files with 918 additions and 0 deletions

View File

@@ -90,6 +90,9 @@ required.
- `FileSystemPDFClient`: Local filesystem PDF access.
- `PDFScraper`: PDF-specific content acquisition.
- `PDFParser`: Base parser for PDF binary interpretation.
- `FileSystemXlsxClient`: Local filesystem spreadsheet access.
- `XlsxScraper`: XLSX-specific content acquisition.
- `XlsxParser`: Generic string-row parser for xlsx workbooks.
---
@@ -108,8 +111,22 @@ required.
"""
from .core import Content, ContentType
from .csv import (
BaseCsvClient,
CsvParser,
CsvParserBase,
CsvScraper,
FileSystemCsvClient,
)
from .html import HTMLScraper, HTMLParser
from .pdf import FileSystemPDFClient, PDFScraper, PDFParser
from .xlsx import (
BaseXlsxClient,
FileSystemXlsxClient,
XlsxParser,
XlsxParserBase,
XlsxScraper,
)
__all__ = [
# core
@@ -124,4 +141,18 @@ __all__ = [
"FileSystemPDFClient",
"PDFScraper",
"PDFParser",
# csv
"BaseCsvClient",
"FileSystemCsvClient",
"CsvScraper",
"CsvParser",
"CsvParserBase",
# xlsx
"BaseXlsxClient",
"FileSystemXlsxClient",
"XlsxScraper",
"XlsxParser",
"XlsxParserBase",
]

View File

@@ -35,6 +35,12 @@ class ContentType(str, Enum):
PDF = "application/pdf"
"""PDF document content."""
XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
"""Office Open XML spreadsheet (xlsx/xlsm) content."""
CSV = "text/csv"
"""Comma-separated-value document content."""
JSON = "application/json"
"""JSON document content."""

26
omniread/csv/__init__.py Normal file
View File

@@ -0,0 +1,26 @@
"""
# Summary
CSV subpackage for OmniRead.
Provides acquisition and parsing of comma-separated-value content:
- `BaseCsvClient`: abstract backing-store client for csv bytes.
- `FileSystemCsvClient`: local filesystem implementation.
- `CsvScraper`: wraps fetched bytes into canonical `Content`.
- `CsvParserBase`: content-type-enforcing parser contract.
- `CsvParser`: generic string-row parser built on the standard csv module.
"""
from .client import BaseCsvClient, FileSystemCsvClient
from .parser import CsvParser
from .parser_base import CsvParserBase
from .scraper import CsvScraper
__all__ = [
"BaseCsvClient",
"FileSystemCsvClient",
"CsvScraper",
"CsvParser",
"CsvParserBase",
]

97
omniread/csv/client.py Normal file
View File

@@ -0,0 +1,97 @@
"""
# Summary
CSV client abstractions for OmniRead.
This module defines the **client layer** responsible for retrieving raw
comma-separated-value document bytes from a concrete backing store.
Clients provide low-level access to csv binaries and are intentionally
decoupled from scraping and parsing logic. They do not perform validation,
interpretation, or content extraction.
Typical backing stores include:
- Local filesystems
- Object storage (S3, GCS, etc.)
- Network file systems
"""
from typing import Any
from abc import ABC, abstractmethod
from pathlib import Path
class BaseCsvClient(ABC):
"""
Abstract client responsible for retrieving csv bytes.
Retrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).
Notes:
**Responsibilities:**
- Implementations must accept a source identifier appropriate to
the backing store.
- Return the full csv binary payload.
- Raise retrieval-specific errors on failure.
"""
@abstractmethod
def fetch(self, source: Any) -> bytes:
"""
Fetch raw csv bytes from the given source.
Args:
source (Any):
Identifier of the csv location, such as a file path,
object storage key, or remote reference.
Returns:
bytes:
Raw csv bytes.
Raises:
Exception:
Retrieval-specific errors defined by the implementation.
"""
raise NotImplementedError
class FileSystemCsvClient(BaseCsvClient):
"""
CSV client that reads from the local filesystem.
Notes:
**Guarantees:**
- This client reads csv files directly from the disk and
returns their raw binary contents.
"""
def fetch(self, path: Path) -> bytes:
"""
Read a csv file from the local filesystem.
Args:
path (Path):
Filesystem path to the csv file.
Returns:
bytes:
Raw csv bytes.
Raises:
FileNotFoundError:
If the path does not exist.
ValueError:
If the path exists but is not a file.
"""
if not path.exists():
raise FileNotFoundError(f"csv not found: {path}")
if not path.is_file():
raise ValueError(f"Path is not a file: {path}")
return path.read_bytes()

102
omniread/csv/parser.py Normal file
View File

@@ -0,0 +1,102 @@
"""
# Summary
CSV parser implementations for OmniRead.
This module provides a concrete, generic parser for comma-separated-value
documents. It exposes records as lists of string cells so downstream
consumers can interpret tabular content without depending on the ``csv``
module directly.
The parser is intentionally statement-agnostic: it performs no header
detection or column interpretation beyond basic cell normalization and
delimiter detection.
"""
from io import StringIO
from csv import Sniffer, reader
from typing import List
from omniread.core.content import Content
from .parser_base import CsvParserBase
class CsvParser(CsvParserBase):
"""
Generic csv parser producing string rows from the document.
Notes:
**Responsibilities:**
- Decode the payload (UTF-8 with BOM support, Latin-1 fallback).
- Detect the delimiter from a leading sample (`,` `;` tab `|`),
defaulting to `,`.
- Normalize cells into deterministic stripped string values.
- Expose row extraction helpers mirroring `XlsxParser.rows`.
**Constraints:**
- All values are strings; consumers requiring typed values must
convert on their side.
- Quoted fields containing delimiters/newlines are handled by
the standard ``csv`` module.
"""
_DELIMITERS = ",;\t|"
def __init__(self, content: Content):
"""
Initialize the parser.
Args:
content (Content):
CSV content to parse; its type must be supported.
"""
super().__init__(content)
def parse(self) -> List[List[str]]:
"""
Parse the document into normalized string rows.
Returns:
List[List[str]]:
Rows of the document.
"""
return self.rows()
def rows(self, *, skip_empty: bool = True) -> List[List[str]]:
"""
Extract normalized string rows from the document.
Args:
skip_empty (bool):
When True (default), rows whose cells are all blank are
omitted.
Returns:
List[List[str]]:
Normalized rows; trailing blank cells are trimmed per row.
"""
text = self._decode(self.content.raw)
dialect_sample = text[:4096]
try:
dialect = Sniffer().sniff(dialect_sample, delimiters=self._DELIMITERS)
delimiter = dialect.delimiter
except Exception:
delimiter = ","
out: List[List[str]] = []
for row in reader(StringIO(text), delimiter=delimiter):
cells = [c.strip() for c in row]
while cells and not cells[-1]:
cells.pop()
if skip_empty and not any(cells):
continue
out.append(cells)
return out
@staticmethod
def _decode(raw: bytes) -> str:
try:
return raw.decode("utf-8-sig")
except UnicodeDecodeError:
return raw.decode("latin-1")

View File

@@ -0,0 +1,55 @@
"""
# Summary
CSV parser base implementation for OmniRead.
This module defines the **CSV-specific parser contract**, extending the
format-agnostic `BaseParser` with constraints appropriate for
comma-separated-value documents.
"""
from typing import Generic, TypeVar
from abc import abstractmethod
from omniread.core.content import ContentType
from omniread.core.parser import BaseParser
T = TypeVar("T")
class CsvParserBase(BaseParser[T], Generic[T]):
"""
Base csv parser.
Notes:
**Responsibilities:**
- This class enforces csv content-type compatibility and provides
the extension point for implementing concrete csv parsing
strategies.
**Constraints:**
- Concrete implementations must define the output type `T` and
implement the `parse()` method.
"""
supported_types = {ContentType.CSV}
"""
Set of content types supported by this parser (CSV only).
"""
@abstractmethod
def parse(self) -> T:
"""
Parse csv content into a structured output.
Returns:
T:
Parsed representation of type `T`.
Raises:
Exception:
Parsing-specific errors as defined by the implementation.
"""
raise NotImplementedError

79
omniread/csv/scraper.py Normal file
View File

@@ -0,0 +1,79 @@
"""
# Summary
CSV scraper for OmniRead.
This module defines the scraper responsible for acquiring raw
comma-separated-value document content from a backing store via a
configured client.
The scraper does not interpret or parse the acquired bytes; it wraps them in
the canonical `Content` model.
"""
from typing import Any, Mapping, Optional
from omniread.core.content import Content, ContentType
from .client import BaseCsvClient
class CsvScraper:
"""
Scraper for csv documents.
Notes:
**Responsibilities:**
- Fetch raw csv bytes via the configured client.
- Wrap the payload in a canonical `Content` instance with the
CSV content type and source identifier.
**Constraints:**
- The scraper does not perform parsing or interpretation.
- Does not assume a specific storage backend.
"""
def __init__(self, *, client: BaseCsvClient):
"""
Initialize the CSV scraper.
Args:
client (BaseCsvClient):
Client responsible for retrieving raw csv bytes.
"""
self._client = client
def fetch(
self,
source: Any,
*,
metadata: Optional[Mapping[str, Any]] = None,
) -> Content:
"""
Fetch a csv document from the given source.
Args:
source (Any):
Identifier of the csv source as understood by the
configured client.
metadata (Optional[Mapping[str, Any]], optional):
Optional metadata to attach to the returned content.
Returns:
Content:
A `Content` instance containing raw csv bytes, source
identifier, CSV content type, and optional metadata.
Raises:
Exception:
Retrieval-specific errors raised by the client.
"""
raw = self._client.fetch(source)
return Content(
raw=raw,
source=source,
content_type=ContentType.CSV,
metadata=dict(metadata) if metadata else None,
)

27
omniread/xlsx/__init__.py Normal file
View File

@@ -0,0 +1,27 @@
"""
# Summary
XLSX subpackage for OmniRead.
Provides acquisition and parsing of Office Open XML spreadsheet (xlsx)
content:
- `BaseXlsxClient`: abstract backing-store client for xlsx bytes.
- `FileSystemXlsxClient`: local filesystem implementation.
- `XlsxScraper`: wraps fetched bytes into canonical `Content`.
- `XlsxParserBase`: content-type-enforcing parser contract.
- `XlsxParser`: generic string-row parser built on openpyxl.
"""
from .client import BaseXlsxClient, FileSystemXlsxClient
from .parser import XlsxParser
from .parser_base import XlsxParserBase
from .scraper import XlsxScraper
__all__ = [
"BaseXlsxClient",
"FileSystemXlsxClient",
"XlsxScraper",
"XlsxParser",
"XlsxParserBase",
]

97
omniread/xlsx/client.py Normal file
View File

@@ -0,0 +1,97 @@
"""
# Summary
XLSX client abstractions for OmniRead.
This module defines the **client layer** responsible for retrieving raw
Office Open XML spreadsheet bytes from a concrete backing store.
Clients provide low-level access to xlsx binaries and are intentionally
decoupled from scraping and parsing logic. They do not perform validation,
interpretation, or content extraction.
Typical backing stores include:
- Local filesystems
- Object storage (S3, GCS, etc.)
- Network file systems
"""
from typing import Any
from abc import ABC, abstractmethod
from pathlib import Path
class BaseXlsxClient(ABC):
"""
Abstract client responsible for retrieving spreadsheet bytes.
Retrieves bytes from a specific backing store (filesystem, S3, FTP, etc.).
Notes:
**Responsibilities:**
- Implementations must accept a source identifier appropriate to
the backing store.
- Return the full xlsx binary payload.
- Raise retrieval-specific errors on failure.
"""
@abstractmethod
def fetch(self, source: Any) -> bytes:
"""
Fetch raw xlsx bytes from the given source.
Args:
source (Any):
Identifier of the spreadsheet location, such as a file path,
object storage key, or remote reference.
Returns:
bytes:
Raw xlsx bytes.
Raises:
Exception:
Retrieval-specific errors defined by the implementation.
"""
raise NotImplementedError
class FileSystemXlsxClient(BaseXlsxClient):
"""
XLSX client that reads from the local filesystem.
Notes:
**Guarantees:**
- This client reads spreadsheet files directly from the disk and
returns their raw binary contents.
"""
def fetch(self, path: Path) -> bytes:
"""
Read an xlsx file from the local filesystem.
Args:
path (Path):
Filesystem path to the spreadsheet file.
Returns:
bytes:
Raw xlsx bytes.
Raises:
FileNotFoundError:
If the path does not exist.
ValueError:
If the path exists but is not a file.
"""
if not path.exists():
raise FileNotFoundError(f"XLSX not found: {path}")
if not path.is_file():
raise ValueError(f"Path is not a file: {path}")
return path.read_bytes()

146
omniread/xlsx/parser.py Normal file
View File

@@ -0,0 +1,146 @@
"""
# Summary
XLSX parser implementations for OmniRead.
This module provides a concrete, generic parser for Office Open XML
spreadsheets. It exposes workbook sheets as lists of string rows so
downstream consumers can interpret tabular content without depending on
openpyxl directly.
The parser is intentionally statement-agnostic: it performs no header
detection or column interpretation beyond basic cell normalization.
"""
import datetime
from io import BytesIO
from typing import List, Optional, Union
import openpyxl
from omniread.core.content import Content
from .parser_base import XlsxParserBase
class XlsxParser(XlsxParserBase):
"""
Generic xlsx parser producing string rows from a worksheet.
Notes:
**Responsibilities:**
- Lazily load the workbook owned by the parser's content.
- Normalize cells (including dates and numeric values) into
deterministic string representations.
- Expose sheet discovery and row extraction helpers.
**Constraints:**
- Cells are rendered with ``str(value)`` after trimming; date and
datetime values are rendered in ISO format. Consumers requiring
locale-specific formatting must convert on their side.
"""
def __init__(self, content: Content, *, data_only: bool = True, read_only: bool = True):
"""
Initialize the parser.
Args:
content (Content):
XLSX content to parse; its type must be supported.
data_only (bool):
Passed to openpyxl: when True, formula cells yield their last
computed value instead of the formula string.
read_only (bool):
Passed to openpyxl: streaming mode for lower memory usage.
"""
super().__init__(content)
self._data_only = data_only
self._read_only = read_only
self._workbook: Optional[openpyxl.Workbook] = None
@property
def workbook(self) -> openpyxl.Workbook:
"""
The lazily loaded workbook backing this parser's content.
"""
if self._workbook is None:
self._workbook = openpyxl.load_workbook(
BytesIO(self.content.raw),
data_only=self._data_only,
read_only=self._read_only,
)
return self._workbook
@property
def sheet_names(self) -> List[str]:
"""
Names of all worksheets contained in the workbook.
"""
return list(self.workbook.sheetnames)
def parse(self) -> List[List[str]]:
"""
Parse the first worksheet into normalized string rows.
Returns:
List[List[str]]:
Rows of the default (first) worksheet.
"""
return self.rows()
def rows(
self,
sheet: Optional[Union[int, str]] = None,
*,
skip_empty: bool = True,
) -> List[List[str]]:
"""
Extract normalized string rows from a worksheet.
Args:
sheet (Optional[Union[int, str]]):
Worksheet index or title; defaults to the first worksheet.
skip_empty (bool):
When True (default), rows whose cells are all blank are omitted.
Returns:
List[List[str]]:
Normalized rows; trailing blank cells are trimmed per row.
Raises:
ValueError:
If the requested sheet does not exist.
"""
ws = self._resolve_sheet(sheet)
out: List[List[str]] = []
for row in ws.iter_rows(values_only=True):
cells = [self._cell_str(v) for v in row]
while cells and not cells[-1].strip():
cells.pop()
if skip_empty and not any(c.strip() for c in cells):
continue
out.append(cells)
return out
def _resolve_sheet(self, sheet: Optional[Union[int, str]]):
names = self.workbook.sheetnames
if not names:
raise ValueError("Workbook contains no worksheets")
if sheet is None:
index = 0
elif isinstance(sheet, int):
index = sheet
else:
index = names.index(sheet) if sheet in names else -1
if not (0 <= index < len(names)):
raise ValueError(f"Worksheet not found: {sheet!r} (available: {names})")
return self.workbook[names[index]]
@staticmethod
def _cell_str(value) -> str:
if value is None:
return ""
if isinstance(value, (datetime.datetime, datetime.date)):
return value.isoformat()
return str(value).strip()

View File

@@ -0,0 +1,55 @@
"""
# Summary
XLSX parser base implementation for OmniRead.
This module defines the **XLSX-specific parser contract**, extending the
format-agnostic `BaseParser` with constraints appropriate for Office Open
XML spreadsheet content.
"""
from typing import Generic, TypeVar
from abc import abstractmethod
from omniread.core.content import ContentType
from omniread.core.parser import BaseParser
T = TypeVar("T")
class XlsxParserBase(BaseParser[T], Generic[T]):
"""
Base xlsx parser.
Notes:
**Responsibilities:**
- This class enforces xlsx content-type compatibility and provides
the extension point for implementing concrete xlsx parsing
strategies.
**Constraints:**
- Concrete implementations must define the output type `T` and
implement the `parse()` method.
"""
supported_types = {ContentType.XLSX}
"""
Set of content types supported by this parser (XLSX only).
"""
@abstractmethod
def parse(self) -> T:
"""
Parse xlsx content into a structured output.
Returns:
T:
Parsed representation of type `T`.
Raises:
Exception:
Parsing-specific errors as defined by the implementation.
"""
raise NotImplementedError

78
omniread/xlsx/scraper.py Normal file
View File

@@ -0,0 +1,78 @@
"""
# Summary
XLSX scraper for OmniRead.
This module defines the scraper responsible for acquiring raw Office Open
XML spreadsheet content from a backing store via a configured client.
The scraper does not interpret or parse the acquired bytes; it wraps them in
the canonical `Content` model.
"""
from typing import Any, Mapping, Optional
from omniread.core.content import Content, ContentType
from .client import BaseXlsxClient
class XlsxScraper:
"""
Scraper for xlsx spreadsheet documents.
Notes:
**Responsibilities:**
- Fetch raw xlsx bytes via the configured client.
- Wrap the payload in a canonical `Content` instance with the
XLSX content type and source identifier.
**Constraints:**
- The scraper does not perform parsing or interpretation.
- Does not assume a specific storage backend.
"""
def __init__(self, *, client: BaseXlsxClient):
"""
Initialize the XLSX scraper.
Args:
client (BaseXlsxClient):
Client responsible for retrieving raw spreadsheet bytes.
"""
self._client = client
def fetch(
self,
source: Any,
*,
metadata: Optional[Mapping[str, Any]] = None,
) -> Content:
"""
Fetch an xlsx document from the given source.
Args:
source (Any):
Identifier of the spreadsheet source as understood by the
configured client.
metadata (Optional[Mapping[str, Any]], optional):
Optional metadata to attach to the returned content.
Returns:
Content:
A `Content` instance containing raw xlsx bytes, source
identifier, XLSX content type, and optional metadata.
Raises:
Exception:
Retrieval-specific errors raised by the client.
"""
raw = self._client.fetch(source)
return Content(
raw=raw,
source=source,
content_type=ContentType.XLSX,
metadata=dict(metadata) if metadata else None,
)

View File

@@ -48,6 +48,7 @@ dependencies = [
"beautifulsoup4>=4.12.0",
# "lxml>=5.0.0",
"pypdf>=4.0.0",
"openpyxl>=3.1.0",
]
[project.optional-dependencies]

118
tests/test_xlsx_simple.py Normal file
View File

@@ -0,0 +1,118 @@
import datetime
from io import BytesIO
import openpyxl
import pytest
from omniread import (
# core
Content,
ContentType,
# xlsx
FileSystemXlsxClient,
XlsxParser,
XlsxScraper,
)
def _fixture_xlsx_bytes() -> bytes:
wb = openpyxl.Workbook()
ws = wb.active
ws.title = "Statement"
ws.append(["DETAILED STATEMENT", None, None, ""])
ws.append([])
ws.append(["S No", "Value Date", "Txn Date", "Remarks", "Withdrawal", "Deposit", "Balance"])
ws.append([1, "01/06/2026", "01/06/2026", "UPI/NPCI BHIM/bhimcashback@h/B", 0.0, 10.0, 250285.24])
row = 4
ws.cell(row=row, column=5).value = 100.0 # ensure float rendering path
buf = BytesIO()
wb.save(buf)
return buf.getvalue()
@pytest.fixture
def xlsx_bytes() -> bytes:
return _fixture_xlsx_bytes()
@pytest.fixture
def xlsx_path(tmp_path, xlsx_bytes):
p = tmp_path / "statement.xlsx"
p.write_bytes(xlsx_bytes)
return p
def test_content_type_value():
assert ContentType.XLSX.value == (
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
)
def test_scraper_round_trip(xlsx_path, xlsx_bytes):
scraper = XlsxScraper(client=FileSystemXlsxClient())
content = scraper.fetch(xlsx_path)
assert content.raw == xlsx_bytes
assert content.content_type is ContentType.XLSX
def test_client_missing_file_raises(tmp_path):
with pytest.raises(FileNotFoundError):
FileSystemXlsxClient().fetch(tmp_path / "nope.xlsx")
def test_client_directory_raises(tmp_path):
with pytest.raises(ValueError):
FileSystemXlsxClient().fetch(tmp_path)
def test_parser_rows_skip_empty_and_trim(xlsx_bytes):
parser = XlsxParser(Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX))
rows = parser.rows()
# empty row dropped by skip_empty; metadata + header + data remain
assert len(rows) == 3
assert rows[0] == ["DETAILED STATEMENT"]
header = rows[1]
assert header[:7] == ["S No", "Value Date", "Txn Date", "Remarks", "Withdrawal", "Deposit", "Balance"]
data = rows[2]
assert data[3] == "UPI/NPCI BHIM/bhimcashback@h/B"
assert data[6] == "250285.24"
def test_parser_keep_empty_rows(xlsx_bytes):
parser = XlsxParser(Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX))
rows = parser.rows(skip_empty=False)
assert len(rows) == 4 # includes blank second row and padded metadata row
def test_parser_sheet_selection_and_names(xlsx_bytes):
parser = XlsxParser(Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX))
assert parser.sheet_names == ["Statement"]
assert parser.rows(sheet="Statement") == parser.rows(sheet=0)
assert parser.parse() == parser.rows()
with pytest.raises(ValueError, match="not found"):
parser.rows(sheet="Missing")
def test_parser_date_cell_isoformat():
wb = openpyxl.Workbook()
ws = wb.active
ws.append([datetime.date(2026, 6, 1), "x"])
buf = BytesIO()
wb.save(buf)
parser = XlsxParser(Content(raw=buf.getvalue(), source="mem", content_type=ContentType.XLSX))
# openpyxl surfaces date cells as datetime; ISO rendering keeps determinism
assert parser.parse()[0][0] == "2026-06-01T00:00:00"
def test_parser_rejects_non_xlsx_content():
with pytest.raises(ValueError, match="does not support"):
XlsxParser(Content(raw=b"<html></html>", source="mem", content_type=ContentType.HTML))