From 1e7e7c8a6ce1c6a0324a2e6beb377588551d5fd2 Mon Sep 17 00:00:00 2001 From: Vishesh 'ironeagle' Bangotra Date: Thu, 10 Sep 2026 18:49:06 +0530 Subject: [PATCH] standardize packaging, tooling, docs, CI, and licensing --- .drone.yml | 22 ++++++++++- CHANGELOG.md | 21 ++++++++++ LICENSE | 21 ++++++++++ omniread/__init__.py | 4 -- omniread/__init__.pyi | 4 +- omniread/core/content.py | 9 +++-- omniread/core/content.pyi | 17 ++++++-- omniread/core/parser.py | 4 +- omniread/core/parser.pyi | 5 ++- omniread/core/scraper.py | 5 ++- omniread/core/scraper.pyi | 8 +++- omniread/csv/client.py | 2 +- omniread/csv/parser.py | 12 +++--- omniread/csv/parser_base.py | 2 +- omniread/csv/scraper.py | 6 ++- omniread/html/__init__.py | 1 - omniread/html/__init__.pyi | 2 +- omniread/html/parser.py | 11 ++---- omniread/html/parser.pyi | 10 +++-- omniread/html/scraper.py | 12 +++--- omniread/html/scraper.pyi | 18 +++++++-- omniread/pdf/__init__.pyi | 2 +- omniread/pdf/client.py | 2 +- omniread/pdf/parser.py | 2 +- omniread/pdf/parser.pyi | 1 + omniread/pdf/scraper.py | 5 ++- omniread/pdf/scraper.pyi | 11 ++++-- omniread/py.typed | 0 omniread/xlsx/client.py | 2 +- omniread/xlsx/parser.py | 25 ++++++------ omniread/xlsx/parser_base.py | 2 +- omniread/xlsx/scraper.py | 6 ++- pyproject.toml | 75 ++++++++++++++++++++++-------------- tests/conftest.py | 16 ++++---- tests/test_html_simple.py | 13 +++---- tests/test_html_table.py | 5 +-- tests/test_pdf_simple.py | 3 +- tests/test_xlsx_simple.py | 55 +++++++++++++++++++++----- 38 files changed, 283 insertions(+), 138 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 LICENSE create mode 100644 omniread/py.typed diff --git a/.drone.yml b/.drone.yml index ddd0ac4..094db62 100644 --- a/.drone.yml +++ b/.drone.yml @@ -32,6 +32,26 @@ steps: echo "🆕 New version detected: $PACKAGE_NAME==$VERSION" fi + - name: quality-gate + image: python:3.13-slim + environment: + PIP_REPO_URL: + from_secret: PIP_REPO_URL + PIP_USERNAME: + from_secret: PIP_USERNAME + PIP_PASSWORD: + from_secret: PIP_PASSWORD + commands: + - pip install --upgrade pip build + - | + AUTH_URL="https://${PIP_USERNAME}:${PIP_PASSWORD}@$(echo "${PIP_REPO_URL#*://}" | sed 's:/*$::')/simple" + pip install --index-url "$AUTH_URL" --extra-index-url https://pypi.org/simple/ -U ".[dev]" + - echo "🛡️ Running quality gate..." + - python -m black --check . + - python -m ruff check . + - python -m mypy + - python -m pytest + - name: build-package image: python:3.13-slim commands: @@ -126,4 +146,4 @@ steps: trigger: event: - - custom + - custom \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..2871ac0 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,21 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added +- `py.typed` marker for PEP 561 type information. +- `.drone.yml` CI with a quality-gate step (black, ruff, mypy, pytest). +- Canonical `docforge.nav.yml`, generated `mkdocs.yml` and `mcp_docs/` via doc-forge. +- MIT `LICENSE`. + +### Changed +- Standardized `pyproject.toml` (canonical packaging, lint tool config, extras). + +### Fixed +- `XLSX`/`CSV` members added to the `content` type surface. +- Stub fixes for typed API surfaces. \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..87328b2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Aetoskia Platform + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/omniread/__init__.py b/omniread/__init__.py index 46f53da..0875478 100644 --- a/omniread/__init__.py +++ b/omniread/__init__.py @@ -132,23 +132,19 @@ __all__ = [ # core "Content", "ContentType", - # html "HTMLScraper", "HTMLParser", - # pdf "FileSystemPDFClient", "PDFScraper", "PDFParser", - # csv "BaseCsvClient", "FileSystemCsvClient", "CsvScraper", "CsvParser", "CsvParserBase", - # xlsx "BaseXlsxClient", "FileSystemXlsxClient", diff --git a/omniread/__init__.pyi b/omniread/__init__.pyi index 0980e93..3699810 100644 --- a/omniread/__init__.pyi +++ b/omniread/__init__.pyi @@ -1,6 +1,6 @@ from .core import Content, ContentType -from .html import HTMLScraper, HTMLParser -from .pdf import FileSystemPDFClient, PDFScraper, PDFParser +from .html import HTMLParser, HTMLScraper +from .pdf import FileSystemPDFClient, PDFParser, PDFScraper __all__ = [ "Content", diff --git a/omniread/core/content.py b/omniread/core/content.py index 775a843..21074b5 100644 --- a/omniread/core/content.py +++ b/omniread/core/content.py @@ -11,9 +11,10 @@ retrieved or parsed. Format-specific behavior and metadata must not alter the semantic meaning of these models. """ -from enum import Enum +from collections.abc import Mapping from dataclasses import dataclass -from typing import Any, Mapping, Optional +from enum import Enum +from typing import Any class ContentType(str, Enum): @@ -72,12 +73,12 @@ class Content: Identifier of the content origin (URL, file path, or logical name). """ - content_type: Optional[ContentType] = None + content_type: ContentType | None = None """ Optional MIME type of the content, if known. """ - metadata: Optional[Mapping[str, Any]] = None + metadata: Mapping[str, Any] | None = None """ Optional, implementation-defined metadata associated with the content (e.g., headers, encoding hints, extraction notes). """ diff --git a/omniread/core/content.pyi b/omniread/core/content.pyi index 5606462..25f6081 100644 --- a/omniread/core/content.pyi +++ b/omniread/core/content.pyi @@ -1,15 +1,24 @@ +from collections.abc import Mapping from enum import Enum -from typing import Any, Mapping, Optional +from typing import Any class ContentType(str, Enum): HTML = "text/html" PDF = "application/pdf" + XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + CSV = "text/csv" JSON = "application/json" XML = "application/xml" class Content: raw: bytes source: str - content_type: Optional[ContentType] - metadata: Optional[Mapping[str, Any]] - def __init__(self, raw: bytes, source: str, content_type: Optional[ContentType] = ..., metadata: Optional[Mapping[str, Any]] = ...) -> None: ... + content_type: ContentType | None + metadata: Mapping[str, Any] | None + def __init__( + self, + raw: bytes, + source: str, + content_type: ContentType | None = ..., + metadata: Mapping[str, Any] | None = ..., + ) -> None: ... diff --git a/omniread/core/parser.py b/omniread/core/parser.py index 7c32993..32baeb2 100644 --- a/omniread/core/parser.py +++ b/omniread/core/parser.py @@ -20,7 +20,7 @@ Parsers are not responsible for: """ from abc import ABC, abstractmethod -from typing import Generic, TypeVar, Set +from typing import Generic, TypeVar from .content import Content, ContentType @@ -46,7 +46,7 @@ class BaseParser(ABC, Generic[T]): - Implementations must remain deterministic for a given input. """ - supported_types: Set[ContentType] = set() + supported_types: set[ContentType] = set() """ Set of content types supported by this parser. An empty set indicates that the parser is content-type agnostic. """ diff --git a/omniread/core/parser.pyi b/omniread/core/parser.pyi index c76e60f..a731065 100644 --- a/omniread/core/parser.pyi +++ b/omniread/core/parser.pyi @@ -1,11 +1,12 @@ from abc import ABC, abstractmethod -from typing import Generic, TypeVar, Set +from typing import Generic, TypeVar + from .content import Content, ContentType T = TypeVar("T") class BaseParser(ABC, Generic[T]): - supported_types: Set[ContentType] + supported_types: set[ContentType] content: Content def __init__(self, content: Content) -> None: ... @abstractmethod diff --git a/omniread/core/scraper.py b/omniread/core/scraper.py index fd4bd81..75dbead 100644 --- a/omniread/core/scraper.py +++ b/omniread/core/scraper.py @@ -22,7 +22,8 @@ All interpretation must be delegated to parsers. """ from abc import ABC, abstractmethod -from typing import Any, Mapping, Optional +from collections.abc import Mapping +from typing import Any from .content import Content @@ -53,7 +54,7 @@ class BaseScraper(ABC): self, source: str, *, - metadata: Optional[Mapping[str, Any]] = None, + metadata: Mapping[str, Any] | None = None, ) -> Content: """ Fetch raw content from the given source. diff --git a/omniread/core/scraper.pyi b/omniread/core/scraper.pyi index cf0ee5f..18d4405 100644 --- a/omniread/core/scraper.pyi +++ b/omniread/core/scraper.pyi @@ -1,7 +1,11 @@ from abc import ABC, abstractmethod -from typing import Any, Mapping, Optional +from collections.abc import Mapping +from typing import Any + from .content import Content class BaseScraper(ABC): @abstractmethod - def fetch(self, source: str, *, metadata: Optional[Mapping[str, Any]] = ...) -> Content: ... + def fetch( + self, source: str, *, metadata: Mapping[str, Any] | None = ... + ) -> Content: ... diff --git a/omniread/csv/client.py b/omniread/csv/client.py index 22ad75f..e373b81 100644 --- a/omniread/csv/client.py +++ b/omniread/csv/client.py @@ -17,9 +17,9 @@ Typical backing stores include: - Network file systems """ -from typing import Any from abc import ABC, abstractmethod from pathlib import Path +from typing import Any class BaseCsvClient(ABC): diff --git a/omniread/csv/parser.py b/omniread/csv/parser.py index 358e664..d13298c 100644 --- a/omniread/csv/parser.py +++ b/omniread/csv/parser.py @@ -13,15 +13,15 @@ detection or column interpretation beyond basic cell normalization and delimiter detection. """ -from io import StringIO from csv import Sniffer, reader -from typing import List +from io import StringIO from omniread.core.content import Content + from .parser_base import CsvParserBase -class CsvParser(CsvParserBase): +class CsvParser(CsvParserBase[list[list[str]]]): """ Generic csv parser producing string rows from the document. @@ -54,7 +54,7 @@ class CsvParser(CsvParserBase): """ super().__init__(content) - def parse(self) -> List[List[str]]: + def parse(self) -> list[list[str]]: """ Parse the document into normalized string rows. @@ -64,7 +64,7 @@ class CsvParser(CsvParserBase): """ return self.rows() - def rows(self, *, skip_empty: bool = True) -> List[List[str]]: + def rows(self, *, skip_empty: bool = True) -> list[list[str]]: """ Extract normalized string rows from the document. @@ -84,7 +84,7 @@ class CsvParser(CsvParserBase): delimiter = dialect.delimiter except Exception: delimiter = "," - out: List[List[str]] = [] + out: list[list[str]] = [] for row in reader(StringIO(text), delimiter=delimiter): cells = [c.strip() for c in row] while cells and not cells[-1]: diff --git a/omniread/csv/parser_base.py b/omniread/csv/parser_base.py index 5daf885..cfa9ec9 100644 --- a/omniread/csv/parser_base.py +++ b/omniread/csv/parser_base.py @@ -8,8 +8,8 @@ format-agnostic `BaseParser` with constraints appropriate for comma-separated-value documents. """ -from typing import Generic, TypeVar from abc import abstractmethod +from typing import Generic, TypeVar from omniread.core.content import ContentType from omniread.core.parser import BaseParser diff --git a/omniread/csv/scraper.py b/omniread/csv/scraper.py index 5785dfc..c729458 100644 --- a/omniread/csv/scraper.py +++ b/omniread/csv/scraper.py @@ -11,9 +11,11 @@ The scraper does not interpret or parse the acquired bytes; it wraps them in the canonical `Content` model. """ -from typing import Any, Mapping, Optional +from collections.abc import Mapping +from typing import Any from omniread.core.content import Content, ContentType + from .client import BaseCsvClient @@ -48,7 +50,7 @@ class CsvScraper: self, source: Any, *, - metadata: Optional[Mapping[str, Any]] = None, + metadata: Mapping[str, Any] | None = None, ) -> Content: """ Fetch a csv document from the given source. diff --git a/omniread/html/__init__.py b/omniread/html/__init__.py index bd9dae0..6922648 100644 --- a/omniread/html/__init__.py +++ b/omniread/html/__init__.py @@ -30,7 +30,6 @@ use this package only when HTML-specific behavior is required. --- """ - from .scraper import HTMLScraper from .parser import HTMLParser diff --git a/omniread/html/__init__.pyi b/omniread/html/__init__.pyi index e52c56d..5bdc4fa 100644 --- a/omniread/html/__init__.pyi +++ b/omniread/html/__init__.pyi @@ -1,4 +1,4 @@ -from .scraper import HTMLScraper from .parser import HTMLParser +from .scraper import HTMLScraper __all__ = ["HTMLScraper", "HTMLParser"] diff --git a/omniread/html/parser.py b/omniread/html/parser.py index de9b49b..d6f244d 100644 --- a/omniread/html/parser.py +++ b/omniread/html/parser.py @@ -16,12 +16,12 @@ Concrete parsers must subclass `HTMLParser` and implement the `parse()` method to return a structured representation appropriate for their use case. """ -from typing import Any, Generic, TypeVar, Optional from abc import abstractmethod +from typing import Any, Generic, TypeVar from bs4 import BeautifulSoup, Tag -from omniread.core.content import ContentType, Content +from omniread.core.content import Content, ContentType from omniread.core.parser import BaseParser T = TypeVar("T") @@ -117,7 +117,7 @@ class HTMLParser(BaseParser[T], Generic[T]): return div.get_text(separator=separator, strip=True) @staticmethod - def parse_link(a: Tag) -> Optional[str]: + def parse_link(a: Tag) -> str | None: """ Extract the hyperlink reference from an `` element. @@ -146,10 +146,7 @@ class HTMLParser(BaseParser[T], Generic[T]): """ rows: list[list[str]] = [] for tr in table.find_all("tr"): - cells = [ - cell.get_text(strip=True) - for cell in tr.find_all(["td", "th"]) - ] + cells = [cell.get_text(strip=True) for cell in tr.find_all(["td", "th"])] if cells: rows.append(cells) return rows diff --git a/omniread/html/parser.pyi b/omniread/html/parser.pyi index 78d8fad..78a7c2d 100644 --- a/omniread/html/parser.pyi +++ b/omniread/html/parser.pyi @@ -1,6 +1,8 @@ -from typing import Any, Generic, TypeVar, Optional, list, dict -from bs4 import BeautifulSoup, Tag -from omniread.core.content import ContentType, Content +from typing import Any, Generic, TypeVar + +from bs4 import Tag + +from omniread.core.content import Content, ContentType from omniread.core.parser import BaseParser T = TypeVar("T") @@ -12,7 +14,7 @@ class HTMLParser(BaseParser[T], Generic[T]): @staticmethod def parse_div(div: Tag, *, separator: str = ...) -> str: ... @staticmethod - def parse_link(a: Tag) -> Optional[str]: ... + def parse_link(a: Tag) -> str | None: ... @staticmethod def parse_table(table: Tag) -> list[list[str]]: ... def parse_meta(self) -> dict[str, Any]: ... diff --git a/omniread/html/scraper.py b/omniread/html/scraper.py index 8d13635..b0bafb7 100644 --- a/omniread/html/scraper.py +++ b/omniread/html/scraper.py @@ -20,8 +20,10 @@ This scraper is not responsible for: - Managing crawl policies or rate limiting """ +from collections.abc import Mapping +from typing import Any + import httpx -from typing import Any, Mapping, Optional from omniread.core.content import Content, ContentType from omniread.core.scraper import BaseScraper @@ -51,7 +53,7 @@ class HTMLScraper(BaseScraper): *, client: httpx.Client | None = None, timeout: float = 15.0, - headers: Optional[Mapping[str, str]] = None, + headers: Mapping[str, str] | None = None, follow_redirects: bool = True, ): """ @@ -97,15 +99,13 @@ class HTMLScraper(BaseScraper): base_ct = raw_ct.split(";", 1)[0].strip().lower() if base_ct != self.content_type.value: - raise ValueError( - f"Expected HTML content, got '{raw_ct}'" - ) + raise ValueError(f"Expected HTML content, got '{raw_ct}'") def fetch( self, source: str, *, - metadata: Optional[Mapping[str, Any]] = None, + metadata: Mapping[str, Any] | None = None, ) -> Content: """ Fetch an HTML document from the given source. diff --git a/omniread/html/scraper.pyi b/omniread/html/scraper.pyi index 2249bca..43838c7 100644 --- a/omniread/html/scraper.pyi +++ b/omniread/html/scraper.pyi @@ -1,10 +1,22 @@ +from collections.abc import Mapping +from typing import Any + import httpx -from typing import Any, Mapping, Optional + from omniread.core.content import Content, ContentType from omniread.core.scraper import BaseScraper class HTMLScraper(BaseScraper): content_type: ContentType - def __init__(self, *, client: Optional[httpx.Client] = ..., timeout: float = ..., headers: Optional[Mapping[str, str]] = ..., follow_redirects: bool = ...) -> None: ... + def __init__( + self, + *, + client: httpx.Client | None = ..., + timeout: float = ..., + headers: Mapping[str, str] | None = ..., + follow_redirects: bool = ..., + ) -> None: ... def validate_content_type(self, response: httpx.Response) -> None: ... - def fetch(self, source: str, *, metadata: Optional[Mapping[str, Any]] = ...) -> Content: ... + def fetch( + self, source: str, *, metadata: Mapping[str, Any] | None = ... + ) -> Content: ... diff --git a/omniread/pdf/__init__.pyi b/omniread/pdf/__init__.pyi index bfd206e..7687f77 100644 --- a/omniread/pdf/__init__.pyi +++ b/omniread/pdf/__init__.pyi @@ -1,5 +1,5 @@ from .client import FileSystemPDFClient -from .scraper import PDFScraper from .parser import PDFParser +from .scraper import PDFScraper __all__ = ["FileSystemPDFClient", "PDFScraper", "PDFParser"] diff --git a/omniread/pdf/client.py b/omniread/pdf/client.py index ee93c64..dd64834 100644 --- a/omniread/pdf/client.py +++ b/omniread/pdf/client.py @@ -17,9 +17,9 @@ Typical backing stores include: - Network file systems """ -from typing import Any from abc import ABC, abstractmethod from pathlib import Path +from typing import Any class BasePDFClient(ABC): diff --git a/omniread/pdf/parser.py b/omniread/pdf/parser.py index 2e010a8..e2c2430 100644 --- a/omniread/pdf/parser.py +++ b/omniread/pdf/parser.py @@ -10,8 +10,8 @@ PDF parsers are responsible for interpreting binary PDF data and producing structured representations suitable for downstream consumption. """ -from typing import Generic, TypeVar from abc import abstractmethod +from typing import Generic, TypeVar from omniread.core.content import ContentType from omniread.core.parser import BaseParser diff --git a/omniread/pdf/parser.pyi b/omniread/pdf/parser.pyi index 79439f7..c81d625 100644 --- a/omniread/pdf/parser.pyi +++ b/omniread/pdf/parser.pyi @@ -1,5 +1,6 @@ from abc import abstractmethod from typing import Generic, TypeVar + from omniread.core.content import ContentType from omniread.core.parser import BaseParser diff --git a/omniread/pdf/scraper.py b/omniread/pdf/scraper.py index 4726580..f94325a 100644 --- a/omniread/pdf/scraper.py +++ b/omniread/pdf/scraper.py @@ -10,7 +10,8 @@ The scraper implements the core `BaseScraper` contract while delegating all storage and access concerns to a `BasePDFClient` implementation. """ -from typing import Any, Mapping, Optional +from collections.abc import Mapping +from typing import Any from omniread.core.content import Content, ContentType from omniread.core.scraper import BaseScraper @@ -48,7 +49,7 @@ class PDFScraper(BaseScraper): self, source: Any, *, - metadata: Optional[Mapping[str, Any]] = None, + metadata: Mapping[str, Any] | None = None, ) -> Content: """ Fetch a PDF document from the given source. diff --git a/omniread/pdf/scraper.pyi b/omniread/pdf/scraper.pyi index cb4ba75..af3fbd5 100644 --- a/omniread/pdf/scraper.pyi +++ b/omniread/pdf/scraper.pyi @@ -1,8 +1,13 @@ -from typing import Any, Mapping, Optional -from omniread.core.content import Content, ContentType +from collections.abc import Mapping +from typing import Any + +from omniread.core.content import Content from omniread.core.scraper import BaseScraper + from .client import BasePDFClient class PDFScraper(BaseScraper): def __init__(self, *, client: BasePDFClient) -> None: ... - def fetch(self, source: Any, *, metadata: Optional[Mapping[str, Any]] = ...) -> Content: ... + def fetch( + self, source: Any, *, metadata: Mapping[str, Any] | None = ... + ) -> Content: ... diff --git a/omniread/py.typed b/omniread/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/omniread/xlsx/client.py b/omniread/xlsx/client.py index b7c9409..a3c67e3 100644 --- a/omniread/xlsx/client.py +++ b/omniread/xlsx/client.py @@ -17,9 +17,9 @@ Typical backing stores include: - Network file systems """ -from typing import Any from abc import ABC, abstractmethod from pathlib import Path +from typing import Any class BaseXlsxClient(ABC): diff --git a/omniread/xlsx/parser.py b/omniread/xlsx/parser.py index bb70288..56ba8dc 100644 --- a/omniread/xlsx/parser.py +++ b/omniread/xlsx/parser.py @@ -14,15 +14,16 @@ detection or column interpretation beyond basic cell normalization. import datetime from io import BytesIO -from typing import List, Optional, Union +from typing import Any import openpyxl from omniread.core.content import Content + from .parser_base import XlsxParserBase -class XlsxParser(XlsxParserBase): +class XlsxParser(XlsxParserBase[list[list[str]]]): """ Generic xlsx parser producing string rows from a worksheet. @@ -41,7 +42,9 @@ class XlsxParser(XlsxParserBase): locale-specific formatting must convert on their side. """ - def __init__(self, content: Content, *, data_only: bool = True, read_only: bool = True): + def __init__( + self, content: Content, *, data_only: bool = True, read_only: bool = True + ): """ Initialize the parser. @@ -57,7 +60,7 @@ class XlsxParser(XlsxParserBase): super().__init__(content) self._data_only = data_only self._read_only = read_only - self._workbook: Optional[openpyxl.Workbook] = None + self._workbook: openpyxl.Workbook | None = None @property def workbook(self) -> openpyxl.Workbook: @@ -73,13 +76,13 @@ class XlsxParser(XlsxParserBase): return self._workbook @property - def sheet_names(self) -> List[str]: + def sheet_names(self) -> list[str]: """ Names of all worksheets contained in the workbook. """ return list(self.workbook.sheetnames) - def parse(self) -> List[List[str]]: + def parse(self) -> list[list[str]]: """ Parse the first worksheet into normalized string rows. @@ -91,10 +94,10 @@ class XlsxParser(XlsxParserBase): def rows( self, - sheet: Optional[Union[int, str]] = None, + sheet: int | str | None = None, *, skip_empty: bool = True, - ) -> List[List[str]]: + ) -> list[list[str]]: """ Extract normalized string rows from a worksheet. @@ -113,7 +116,7 @@ class XlsxParser(XlsxParserBase): If the requested sheet does not exist. """ ws = self._resolve_sheet(sheet) - out: List[List[str]] = [] + out: list[list[str]] = [] for row in ws.iter_rows(values_only=True): cells = [self._cell_str(v) for v in row] while cells and not cells[-1].strip(): @@ -123,7 +126,7 @@ class XlsxParser(XlsxParserBase): out.append(cells) return out - def _resolve_sheet(self, sheet: Optional[Union[int, str]]): + def _resolve_sheet(self, sheet: int | str | None) -> Any: names = self.workbook.sheetnames if not names: raise ValueError("Workbook contains no worksheets") @@ -138,7 +141,7 @@ class XlsxParser(XlsxParserBase): return self.workbook[names[index]] @staticmethod - def _cell_str(value) -> str: + def _cell_str(value: Any) -> str: if value is None: return "" if isinstance(value, (datetime.datetime, datetime.date)): diff --git a/omniread/xlsx/parser_base.py b/omniread/xlsx/parser_base.py index 9974ef6..06acfbf 100644 --- a/omniread/xlsx/parser_base.py +++ b/omniread/xlsx/parser_base.py @@ -8,8 +8,8 @@ format-agnostic `BaseParser` with constraints appropriate for Office Open XML spreadsheet content. """ -from typing import Generic, TypeVar from abc import abstractmethod +from typing import Generic, TypeVar from omniread.core.content import ContentType from omniread.core.parser import BaseParser diff --git a/omniread/xlsx/scraper.py b/omniread/xlsx/scraper.py index 00202ad..c125160 100644 --- a/omniread/xlsx/scraper.py +++ b/omniread/xlsx/scraper.py @@ -10,9 +10,11 @@ The scraper does not interpret or parse the acquired bytes; it wraps them in the canonical `Content` model. """ -from typing import Any, Mapping, Optional +from collections.abc import Mapping +from typing import Any from omniread.core.content import Content, ContentType + from .client import BaseXlsxClient @@ -47,7 +49,7 @@ class XlsxScraper: self, source: Any, *, - metadata: Optional[Mapping[str, Any]] = None, + metadata: Mapping[str, Any] | None = None, ) -> Content: """ Fetch an xlsx document from the given source. diff --git a/pyproject.toml b/pyproject.toml index fb78aff..df6d018 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,14 @@ [build-system] -requires = ["setuptools>=65.0", "wheel"] +requires = ["setuptools>=68", "wheel"] build-backend = "setuptools.build_meta" + [project] name = "omniread" version = "0.0.1" description = "Composable content ingestion framework with pluggable scrapers and parsers for HTML, PDF, and structured data" readme = "README.md" -requires-python = ">=3.9" +requires-python = ">=3.10" license = { text = "MIT" } authors = [ @@ -17,6 +18,7 @@ maintainers = [ { name = "Aetos Skia", email = "dev@aetoskia.com" } ] + keywords = [ "scraping", "parsing", @@ -33,39 +35,49 @@ classifiers = [ "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", - "Topic :: Security", "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Text Processing", "Typing :: Typed", ] + dependencies = [ "requests>=2.31.0", "beautifulsoup4>=4.12.0", -# "lxml>=5.0.0", "pypdf>=4.0.0", "openpyxl>=3.1.0", ] [project.optional-dependencies] dev = [ - "pytest>=7.4.0", + "pytest>=8.0.0", "pytest-asyncio>=0.21.0", "pytest-cov>=4.1.0", "black>=23.0.0", - "ruff>=0.1.0", - "mypy>=1.5.0", + "ruff>=0.3.0", + "mypy>=1.8.0", + "build>=1.0.0", + "twine>=4.0.0", "pre-commit>=3.4.0", + "doc-forge[mcp,mkdocs]>=0.0.6", +] + +docs = [ + "mkdocs>=1.5.0", + "mkdocs-material>=9.5.0", + "mkdocstrings[python]>=0.24.0", ] all = [ - "omniread[dev,fastapi]", + "omniread[dev,docs]", ] + [project.urls] Homepage = "https://git.aetoskia.com/aetos/omniread" Documentation = "https://git.aetoskia.com/aetos/omniread#readme" @@ -73,12 +85,14 @@ Repository = "https://git.aetoskia.com/aetos/omniread.git" Issues = "https://git.aetoskia.com/aetos/omniread/issues" Versions = "https://git.aetoskia.com/aetos/omniread/tags" + [tool.setuptools] packages = { find = { include = ["omniread*"] } } [tool.setuptools.package-data] omniread = ["py.typed"] + [tool.pytest.ini_options] testpaths = ["tests"] asyncio_mode = "auto" @@ -94,9 +108,10 @@ addopts = [ "--cov-report=xml", ] + [tool.black] line-length = 88 -target-version = ["py39", "py310", "py311", "py312", "py313"] +target-version = ["py310", "py311", "py312", "py313"] include = '\.pyi?$' extend-exclude = ''' /( @@ -111,9 +126,12 @@ extend-exclude = ''' )/ ''' + [tool.ruff] line-length = 88 -target-version = "py39" +target-version = "py310" + +[tool.ruff.lint] select = [ "E", "W", @@ -129,31 +147,30 @@ ignore = [ "C901", ] -[tool.ruff.per-file-ignores] -"__init__.py" = ["F401"] +[tool.ruff.lint.per-file-ignores] +"__init__.py" = ["F401", "I001"] +"tests/*" = ["B008"] + [tool.mypy] -python_version = "3.9" -warn_return_any = true -warn_unused_configs = true -disallow_untyped_defs = true -disallow_incomplete_defs = true -check_untyped_defs = true -disallow_untyped_decorators = false -no_implicit_optional = true -warn_redundant_casts = true -warn_unused_ignores = true -warn_no_return = true -follow_imports = "normal" -strict_optional = true +python_version = "3.10" +strict = true +exclude = [ + "tests/", +] +files = ["omniread"] [[tool.mypy.overrides]] module = [ - "jose.*", - "httpx.*", + "requests.*", + "bs4.*", + "lxml.*", + "pypdf.*", + "openpyxl.*", ] ignore_missing_imports = true + [tool.coverage.run] source = ["omniread"] omit = [ @@ -170,4 +187,4 @@ exclude_lines = [ "raise NotImplementedError", "if TYPE_CHECKING:", "@abstractmethod", -] +] \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 309c278..2a35ef2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,22 +1,20 @@ import json -import pytest -import httpx from pathlib import Path -from jinja2 import Environment, BaseLoader + +import httpx +import pytest +from jinja2 import BaseLoader, Environment from omniread import ( # core ContentType, - - # html - HTMLScraper, - # pdf FileSystemPDFClient, + # html + HTMLScraper, PDFScraper, ) - MOCK_HTML_DIR = Path(__file__).parent / "mocks" / "html" MOCK_PDF_DIR = Path(__file__).parent / "mocks" / "pdf" @@ -40,7 +38,7 @@ def mock_transport(request: httpx.Request) -> httpx.Response: httpx MockTransport handler. """ path = request.url.path - if path not in ['/simple', '/table']: + if path not in ["/simple", "/table"]: return httpx.Response( status_code=404, content=b"Not Found", diff --git a/tests/test_html_simple.py b/tests/test_html_simple.py index cd32f0c..9ec751a 100644 --- a/tests/test_html_simple.py +++ b/tests/test_html_simple.py @@ -1,22 +1,19 @@ -from typing import Optional - -from pydantic import BaseModel from bs4 import Tag +from pydantic import BaseModel from omniread import ( # core Content, - # html HTMLParser, ) class ParsedSimpleHTML(BaseModel): - title: Optional[str] - description: Optional[str] - content: Optional[str] - link: Optional[str] + title: str | None + description: str | None + content: str | None + link: str | None class SimpleHTMLParser(HTMLParser[ParsedSimpleHTML]): diff --git a/tests/test_html_table.py b/tests/test_html_table.py index dd4a133..b6ad656 100644 --- a/tests/test_html_table.py +++ b/tests/test_html_table.py @@ -1,18 +1,15 @@ -from typing import Optional - from pydantic import BaseModel from omniread import ( # core Content, - # html HTMLParser, ) class ParsedTableHTML(BaseModel): - title: Optional[str] + title: str | None table: list[list[str]] diff --git a/tests/test_pdf_simple.py b/tests/test_pdf_simple.py index b90f791..cd6f82e 100644 --- a/tests/test_pdf_simple.py +++ b/tests/test_pdf_simple.py @@ -1,14 +1,15 @@ from typing import Literal + from pydantic import BaseModel from omniread import ( # core Content, - # pdf PDFParser, ) + class ParsedPDF(BaseModel): size_bytes: int magic: Literal[b"%PDF"] diff --git a/tests/test_xlsx_simple.py b/tests/test_xlsx_simple.py index 1390b19..4acaa42 100644 --- a/tests/test_xlsx_simple.py +++ b/tests/test_xlsx_simple.py @@ -8,7 +8,6 @@ from omniread import ( # core Content, ContentType, - # xlsx FileSystemXlsxClient, XlsxParser, @@ -22,8 +21,28 @@ def _fixture_xlsx_bytes() -> bytes: ws.title = "Statement" ws.append(["DETAILED STATEMENT", None, None, ""]) ws.append([]) - ws.append(["S No", "Value Date", "Txn Date", "Remarks", "Withdrawal", "Deposit", "Balance"]) - ws.append([1, "01/06/2026", "01/06/2026", "UPI/NPCI BHIM/bhimcashback@h/B", 0.0, 10.0, 250285.24]) + ws.append( + [ + "S No", + "Value Date", + "Txn Date", + "Remarks", + "Withdrawal", + "Deposit", + "Balance", + ] + ) + ws.append( + [ + 1, + "01/06/2026", + "01/06/2026", + "UPI/NPCI BHIM/bhimcashback@h/B", + 0.0, + 10.0, + 250285.24, + ] + ) row = 4 ws.cell(row=row, column=5).value = 100.0 # ensure float rendering path buf = BytesIO() @@ -68,7 +87,9 @@ def test_client_directory_raises(tmp_path): def test_parser_rows_skip_empty_and_trim(xlsx_bytes): - parser = XlsxParser(Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX)) + parser = XlsxParser( + Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX) + ) rows = parser.rows() @@ -76,14 +97,24 @@ def test_parser_rows_skip_empty_and_trim(xlsx_bytes): assert len(rows) == 3 assert rows[0] == ["DETAILED STATEMENT"] header = rows[1] - assert header[:7] == ["S No", "Value Date", "Txn Date", "Remarks", "Withdrawal", "Deposit", "Balance"] + assert header[:7] == [ + "S No", + "Value Date", + "Txn Date", + "Remarks", + "Withdrawal", + "Deposit", + "Balance", + ] data = rows[2] assert data[3] == "UPI/NPCI BHIM/bhimcashback@h/B" assert data[6] == "250285.24" def test_parser_keep_empty_rows(xlsx_bytes): - parser = XlsxParser(Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX)) + parser = XlsxParser( + Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX) + ) rows = parser.rows(skip_empty=False) @@ -91,7 +122,9 @@ def test_parser_keep_empty_rows(xlsx_bytes): def test_parser_sheet_selection_and_names(xlsx_bytes): - parser = XlsxParser(Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX)) + parser = XlsxParser( + Content(raw=xlsx_bytes, source="mem", content_type=ContentType.XLSX) + ) assert parser.sheet_names == ["Statement"] assert parser.rows(sheet="Statement") == parser.rows(sheet=0) @@ -108,11 +141,15 @@ def test_parser_date_cell_isoformat(): buf = BytesIO() wb.save(buf) - parser = XlsxParser(Content(raw=buf.getvalue(), source="mem", content_type=ContentType.XLSX)) + parser = XlsxParser( + Content(raw=buf.getvalue(), source="mem", content_type=ContentType.XLSX) + ) # openpyxl surfaces date cells as datetime; ISO rendering keeps determinism assert parser.parse()[0][0] == "2026-06-01T00:00:00" def test_parser_rejects_non_xlsx_content(): with pytest.raises(ValueError, match="does not support"): - XlsxParser(Content(raw=b"", source="mem", content_type=ContentType.HTML)) + XlsxParser( + Content(raw=b"", source="mem", content_type=ContentType.HTML) + )