standardize packaging, tooling, docs, CI, and licensing
This commit is contained in:
@@ -1,22 +1,20 @@
|
||||
import json
|
||||
import pytest
|
||||
import httpx
|
||||
from pathlib import Path
|
||||
from jinja2 import Environment, BaseLoader
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from jinja2 import BaseLoader, Environment
|
||||
|
||||
from omniread import (
|
||||
# core
|
||||
ContentType,
|
||||
|
||||
# html
|
||||
HTMLScraper,
|
||||
|
||||
# pdf
|
||||
FileSystemPDFClient,
|
||||
# html
|
||||
HTMLScraper,
|
||||
PDFScraper,
|
||||
)
|
||||
|
||||
|
||||
MOCK_HTML_DIR = Path(__file__).parent / "mocks" / "html"
|
||||
MOCK_PDF_DIR = Path(__file__).parent / "mocks" / "pdf"
|
||||
|
||||
@@ -40,7 +38,7 @@ def mock_transport(request: httpx.Request) -> httpx.Response:
|
||||
httpx MockTransport handler.
|
||||
"""
|
||||
path = request.url.path
|
||||
if path not in ['/simple', '/table']:
|
||||
if path not in ["/simple", "/table"]:
|
||||
return httpx.Response(
|
||||
status_code=404,
|
||||
content=b"Not Found",
|
||||
|
||||
@@ -1,22 +1,19 @@
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import BaseModel
|
||||
from bs4 import Tag
|
||||
from pydantic import BaseModel
|
||||
|
||||
from omniread import (
|
||||
# core
|
||||
Content,
|
||||
|
||||
# html
|
||||
HTMLParser,
|
||||
)
|
||||
|
||||
|
||||
class ParsedSimpleHTML(BaseModel):
|
||||
title: Optional[str]
|
||||
description: Optional[str]
|
||||
content: Optional[str]
|
||||
link: Optional[str]
|
||||
title: str | None
|
||||
description: str | None
|
||||
content: str | None
|
||||
link: str | None
|
||||
|
||||
|
||||
class SimpleHTMLParser(HTMLParser[ParsedSimpleHTML]):
|
||||
|
||||
@@ -1,18 +1,15 @@
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from omniread import (
|
||||
# core
|
||||
Content,
|
||||
|
||||
# html
|
||||
HTMLParser,
|
||||
)
|
||||
|
||||
|
||||
class ParsedTableHTML(BaseModel):
|
||||
title: Optional[str]
|
||||
title: str | None
|
||||
table: list[list[str]]
|
||||
|
||||
|
||||
|
||||
@@ -1,14 +1,15 @@
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from omniread import (
|
||||
# core
|
||||
Content,
|
||||
|
||||
# pdf
|
||||
PDFParser,
|
||||
)
|
||||
|
||||
|
||||
class ParsedPDF(BaseModel):
|
||||
size_bytes: int
|
||||
magic: Literal[b"%PDF"]
|
||||
|
||||
@@ -8,7 +8,6 @@ from omniread import (
|
||||
# core
|
||||
Content,
|
||||
ContentType,
|
||||
|
||||
# xlsx
|
||||
FileSystemXlsxClient,
|
||||
XlsxParser,
|
||||
@@ -22,8 +21,28 @@ def _fixture_xlsx_bytes() -> bytes:
|
||||
ws.title = "Statement"
|
||||
ws.append(["DETAILED STATEMENT", None, None, ""])
|
||||
ws.append([])
|
||||
ws.append(["S No", "Value Date", "Txn Date", "Remarks", "Withdrawal", "Deposit", "Balance"])
|
||||
ws.append([1, "01/06/2026", "01/06/2026", "UPI/NPCI BHIM/bhimcashback@h/B", 0.0, 10.0, 250285.24])
|
||||
ws.append(
|
||||
[
|
||||
"S No",
|
||||
"Value Date",
|
||||
"Txn Date",
|
||||
"Remarks",
|
||||
"Withdrawal",
|
||||
"Deposit",
|
||||
"Balance",
|
||||
]
|
||||
)
|
||||
ws.append(
|
||||
[
|
||||
1,
|
||||
"01/06/2026",
|
||||
"01/06/2026",
|
||||
"UPI/NPCI BHIM/bhimcashback@h/B",
|
||||
0.0,
|
||||
10.0,
|
||||
250285.24,
|
||||
]
|
||||
)
|
||||
row = 4
|
||||
ws.cell(row=row, column=5).value = 100.0 # ensure float rendering path
|
||||
buf = BytesIO()
|
||||
@@ -68,7 +87,9 @@ def test_client_directory_raises(tmp_path):
|
||||
|
||||
|
||||
def test_parser_rows_skip_empty_and_trim(xlsx_bytes):
|
||||
parser = XlsxParser(Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX))
|
||||
parser = XlsxParser(
|
||||
Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX)
|
||||
)
|
||||
|
||||
rows = parser.rows()
|
||||
|
||||
@@ -76,14 +97,24 @@ def test_parser_rows_skip_empty_and_trim(xlsx_bytes):
|
||||
assert len(rows) == 3
|
||||
assert rows[0] == ["DETAILED STATEMENT"]
|
||||
header = rows[1]
|
||||
assert header[:7] == ["S No", "Value Date", "Txn Date", "Remarks", "Withdrawal", "Deposit", "Balance"]
|
||||
assert header[:7] == [
|
||||
"S No",
|
||||
"Value Date",
|
||||
"Txn Date",
|
||||
"Remarks",
|
||||
"Withdrawal",
|
||||
"Deposit",
|
||||
"Balance",
|
||||
]
|
||||
data = rows[2]
|
||||
assert data[3] == "UPI/NPCI BHIM/bhimcashback@h/B"
|
||||
assert data[6] == "250285.24"
|
||||
|
||||
|
||||
def test_parser_keep_empty_rows(xlsx_bytes):
|
||||
parser = XlsxParser(Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX))
|
||||
parser = XlsxParser(
|
||||
Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX)
|
||||
)
|
||||
|
||||
rows = parser.rows(skip_empty=False)
|
||||
|
||||
@@ -91,7 +122,9 @@ def test_parser_keep_empty_rows(xlsx_bytes):
|
||||
|
||||
|
||||
def test_parser_sheet_selection_and_names(xlsx_bytes):
|
||||
parser = XlsxParser(Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX))
|
||||
parser = XlsxParser(
|
||||
Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX)
|
||||
)
|
||||
|
||||
assert parser.sheet_names == ["Statement"]
|
||||
assert parser.rows(sheet="Statement") == parser.rows(sheet=0)
|
||||
@@ -108,11 +141,15 @@ def test_parser_date_cell_isoformat():
|
||||
buf = BytesIO()
|
||||
wb.save(buf)
|
||||
|
||||
parser = XlsxParser(Content(raw=buf.getvalue(), source="mem", content_type=ContentType.XLSX))
|
||||
parser = XlsxParser(
|
||||
Content(raw=buf.getvalue(), source="mem", content_type=ContentType.XLSX)
|
||||
)
|
||||
# openpyxl surfaces date cells as datetime; ISO rendering keeps determinism
|
||||
assert parser.parse()[0][0] == "2026-06-01T00:00:00"
|
||||
|
||||
|
||||
def test_parser_rejects_non_xlsx_content():
|
||||
with pytest.raises(ValueError, match="does not support"):
|
||||
XlsxParser(Content(raw=b"<html></html>", source="mem", content_type=ContentType.HTML))
|
||||
XlsxParser(
|
||||
Content(raw=b"<html></html>", source="mem", content_type=ContentType.HTML)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user