Files
omniread/omniread/csv/parser.py

103 lines
3.0 KiB
Python

"""
# Summary
CSV parser implementations for OmniRead.
This module provides a concrete, generic parser for comma-separated-value
documents. It exposes records as lists of string cells so downstream
consumers can interpret tabular content without depending on the ``csv``
module directly.
The parser is intentionally statement-agnostic: it performs no header
detection or column interpretation beyond basic cell normalization and
delimiter detection.
"""
from csv import Sniffer, reader
from io import StringIO
from omniread.core.content import Content
from .parser_base import CsvParserBase
class CsvParser(CsvParserBase[list[list[str]]]):
"""
Generic csv parser producing string rows from the document.
Notes:
**Responsibilities:**
- Decode the payload (UTF-8 with BOM support, Latin-1 fallback).
- Detect the delimiter from a leading sample (`,` `;` tab `|`),
defaulting to `,`.
- Normalize cells into deterministic stripped string values.
- Expose row extraction helpers mirroring `XlsxParser.rows`.
**Constraints:**
- All values are strings; consumers requiring typed values must
convert on their side.
- Quoted fields containing delimiters/newlines are handled by
the standard ``csv`` module.
"""
_DELIMITERS = ",;\t|"
def __init__(self, content: Content):
"""
Initialize the parser.
Args:
content (Content):
CSV content to parse; its type must be supported.
"""
super().__init__(content)
def parse(self) -> list[list[str]]:
"""
Parse the document into normalized string rows.
Returns:
List[List[str]]:
Rows of the document.
"""
return self.rows()
def rows(self, *, skip_empty: bool = True) -> list[list[str]]:
"""
Extract normalized string rows from the document.
Args:
skip_empty (bool):
When True (default), rows whose cells are all blank are
omitted.
Returns:
List[List[str]]:
Normalized rows; trailing blank cells are trimmed per row.
"""
text = self._decode(self.content.raw)
dialect_sample = text[:4096]
try:
dialect = Sniffer().sniff(dialect_sample, delimiters=self._DELIMITERS)
delimiter = dialect.delimiter
except Exception:
delimiter = ","
out: list[list[str]] = []
for row in reader(StringIO(text), delimiter=delimiter):
cells = [c.strip() for c in row]
while cells and not cells[-1]:
cells.pop()
if skip_empty and not any(cells):
continue
out.append(cells)
return out
@staticmethod
def _decode(raw: bytes) -> str:
try:
return raw.decode("utf-8-sig")
except UnicodeDecodeError:
return raw.decode("latin-1")