103 lines
3.0 KiB
Python
103 lines
3.0 KiB
Python
"""
|
|
# Summary
|
|
|
|
CSV parser implementations for OmniRead.
|
|
|
|
This module provides a concrete, generic parser for comma-separated-value
|
|
documents. It exposes records as lists of string cells so downstream
|
|
consumers can interpret tabular content without depending on the ``csv``
|
|
module directly.
|
|
|
|
The parser is intentionally statement-agnostic: it performs no header
|
|
detection or column interpretation beyond basic cell normalization and
|
|
delimiter detection.
|
|
"""
|
|
|
|
from csv import Sniffer, reader
|
|
from io import StringIO
|
|
|
|
from omniread.core.content import Content
|
|
|
|
from .parser_base import CsvParserBase
|
|
|
|
|
|
class CsvParser(CsvParserBase[list[list[str]]]):
|
|
"""
|
|
Generic csv parser producing string rows from the document.
|
|
|
|
Notes:
|
|
**Responsibilities:**
|
|
|
|
- Decode the payload (UTF-8 with BOM support, Latin-1 fallback).
|
|
- Detect the delimiter from a leading sample (`,` `;` tab `|`),
|
|
defaulting to `,`.
|
|
- Normalize cells into deterministic stripped string values.
|
|
- Expose row extraction helpers mirroring `XlsxParser.rows`.
|
|
|
|
**Constraints:**
|
|
|
|
- All values are strings; consumers requiring typed values must
|
|
convert on their side.
|
|
- Quoted fields containing delimiters/newlines are handled by
|
|
the standard ``csv`` module.
|
|
"""
|
|
|
|
_DELIMITERS = ",;\t|"
|
|
|
|
def __init__(self, content: Content):
|
|
"""
|
|
Initialize the parser.
|
|
|
|
Args:
|
|
content (Content):
|
|
CSV content to parse; its type must be supported.
|
|
"""
|
|
super().__init__(content)
|
|
|
|
def parse(self) -> list[list[str]]:
|
|
"""
|
|
Parse the document into normalized string rows.
|
|
|
|
Returns:
|
|
List[List[str]]:
|
|
Rows of the document.
|
|
"""
|
|
return self.rows()
|
|
|
|
def rows(self, *, skip_empty: bool = True) -> list[list[str]]:
|
|
"""
|
|
Extract normalized string rows from the document.
|
|
|
|
Args:
|
|
skip_empty (bool):
|
|
When True (default), rows whose cells are all blank are
|
|
omitted.
|
|
|
|
Returns:
|
|
List[List[str]]:
|
|
Normalized rows; trailing blank cells are trimmed per row.
|
|
"""
|
|
text = self._decode(self.content.raw)
|
|
dialect_sample = text[:4096]
|
|
try:
|
|
dialect = Sniffer().sniff(dialect_sample, delimiters=self._DELIMITERS)
|
|
delimiter = dialect.delimiter
|
|
except Exception:
|
|
delimiter = ","
|
|
out: list[list[str]] = []
|
|
for row in reader(StringIO(text), delimiter=delimiter):
|
|
cells = [c.strip() for c in row]
|
|
while cells and not cells[-1]:
|
|
cells.pop()
|
|
if skip_empty and not any(cells):
|
|
continue
|
|
out.append(cells)
|
|
return out
|
|
|
|
@staticmethod
|
|
def _decode(raw: bytes) -> str:
|
|
try:
|
|
return raw.decode("utf-8-sig")
|
|
except UnicodeDecodeError:
|
|
return raw.decode("latin-1")
|