pyproject.toml

[build-system]
requires = ["setuptools>=61.0", "wheel"]
build-backend = "setuptools.build_meta"

[project]
name = "iterabledata"
description = "Iterable data processing Python library"
dynamic = ["version", "readme"]
requires-python = ">=3.10"
authors = [{name = "Ivan Begtin"}]
license = {text = "MIT"}
classifiers = [
    "Development Status :: 5 - Production/Stable",
    "Programming Language :: Python",
    "Programming Language :: Python :: 3 :: Only",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Environment :: Console",
    "Intended Audience :: Developers",
    "Intended Audience :: System Administrators",
    "License :: OSI Approved :: MIT License",
    "Topic :: Software Development",
    "Topic :: Text Processing",
]
keywords = ["json", "jsonl", "csv", "bson", "parquet", "orc", "xml", "xls", "xlsx", "dbf", "dataset", "etl", "data-pipelines"]
dependencies = [
    "chardet>=5.0.0",
    "tqdm>=4.64.0",
]

[project.optional-dependencies]
# Format / feature extras (kept optional to minimize base install footprint).
compression = [
    "lz4>=4.0.0",
    "python-snappy>=0.6.0",
    "python-lzo>=1.12.0",
    "brotli>=1.0.0",
    "brotli_file>=1.0.0",
    "zstandard>=0.19.0",
]
parquet = ["pyarrow>=10.0.0"]
orc = ["pyorc>=1.6.0"]
excel = ["xlrd>=2.0.0", "openpyxl>=3.0.0"]
xml = ["lxml>=4.9.0"]
bson = ["pymongo>=4.0.0"]
dbf = ["dbfread>=2.0.0"]
warc = ["warcio>=1.7.0"]
duckdb = ["duckdb>=0.9.0"]
stats = ["pyreadstat>=1.2.0"]
protobuf = ["protobuf>=4.0.0"]
ion = ["ion-python>=1.0.0"]
hdf5 = ["h5py>=3.0.0"]
geospatial = ["geojson>=3.0.0", "pyshp>=2.3.0", "fiona>=1.9.0", "mapbox-vector-tile>=1.2.0", "topojson>=1.5.0"]
toml = ["tomli>=2.0.0", "tomli-w>=1.0.0", "toml>=0.10.0"]
msgpack = ["msgpack>=1.0.0"]
yaml = ["pyyaml>=6.0.0"]
pcap = ["dpkt>=1.9.0"]
netcdf = ["netCDF4>=1.6.0"]
mvt = ["mapbox-vector-tile>=1.2.0"]
topojson = ["topojson>=1.5.0"]
feed = ["feedparser>=6.0.0"]
dxf = ["ezdxf>=1.0.0"]
html = ["beautifulsoup4>=4.11.0"]
arff = ["liac-arff>=2.5.0"]
json = ["ijson>=3.2.0"]
vortex = ["vortex-data>=0.56.0"]
pydantic = ["pydantic>=2.0.0"]
dataframes = ["pandas>=1.5.0", "polars>=0.19.0", "dask[dataframe]>=2023.1.0"]
cloud = ["fsspec>=2023.1.0", "s3fs>=2023.1.0", "gcsfs>=2023.1.0", "adlfs>=2023.1.0"]
# AI-powered documentation
ai = ["openai>=1.0.0", "requests>=2.28.0"]
# Database engines (read-only support)
db = [
    "psycopg2-binary>=2.9.0",  # PostgreSQL
    "pymongo>=4.0.0",  # MongoDB
    "elasticsearch>=8.0.0",  # Elasticsearch/OpenSearch
    "pymysql>=1.0.0",  # MySQL/MariaDB
    "pyodbc>=4.0.0",  # Microsoft SQL Server
    "clickhouse-connect>=0.6.0",  # ClickHouse
]
# SQL databases only
db-sql = [
    "psycopg2-binary>=2.9.0",  # PostgreSQL
    "pymysql>=1.0.0",  # MySQL/MariaDB
    "pyodbc>=4.0.0",  # Microsoft SQL Server
    "clickhouse-connect>=0.6.0",  # ClickHouse
]
# NoSQL databases only
db-nosql = [
    "pymongo>=4.0.0",  # MongoDB
    "elasticsearch>=8.0.0",  # Elasticsearch/OpenSearch
]
# Database ingestion (write support - existing)
db-ingest = [
    "psycopg[binary]>=3.0.0",  # PostgreSQL (psycopg3 for ingestion)
    "pymongo>=4.0.0",  # MongoDB
    "mysql-connector-python>=8.0.0",  # MySQL
    "elasticsearch>=8.0.0",  # Elasticsearch
    "duckdb>=0.9.0",  # DuckDB (also in duckdb extra)
]


# Convenience group
all = [
    "lz4>=4.0.0",
    "python-snappy>=0.6.0",
    "python-lzo>=1.12.0",
    "brotli>=1.0.0",
    "brotli_file>=1.0.0",
    "zstandard>=0.19.0",
    "pyarrow>=10.0.0",
    "pyorc>=1.6.0",
    "xlrd>=2.0.0",
    "openpyxl>=3.0.0",
    "lxml>=4.9.0",
    "pymongo>=4.0.0",
    "dbfread>=2.0.0",
    "warcio>=1.7.0",
    "duckdb>=0.9.0",
    "pyreadstat>=1.2.0",
    "protobuf>=4.0.0",
    "ion-python>=1.0.0",
    "h5py>=3.0.0",
    "geojson>=3.0.0",
    "pyshp>=2.3.0",
    "fiona>=1.9.0",
    "tomli>=2.0.0",
    "tomli-w>=1.0.0",
    "toml>=0.10.0",
    "msgpack>=1.0.0",
    "pyyaml>=6.0.0",
    "dpkt>=1.9.0",
    "netCDF4>=1.6.0",
    "mapbox-vector-tile>=1.2.0",
    "topojson>=1.5.0",
    "feedparser>=6.0.0",
    "ezdxf>=1.0.0",
    "beautifulsoup4>=4.11.0",
    "liac-arff>=2.5.0",
    "ijson>=3.2.0",
    "vortex-data>=0.56.0",
    "pydantic>=2.0.0",
    "pandas>=1.5.0",
    "polars>=0.19.0",
    "dask[dataframe]>=2023.1.0",
    "fsspec>=2023.1.0",
    "s3fs>=2023.1.0",
    "gcsfs>=2023.1.0",
    "adlfs>=2023.1.0",
    # AI and database extras
    "openai>=1.0.0",
    "requests>=2.28.0",
    # Database engines (read-only)
    "psycopg2-binary>=2.9.0",
    "pymongo>=4.0.0",
    "elasticsearch>=8.0.0",
    "pymysql>=1.0.0",
    "pyodbc>=4.0.0",
    "clickhouse-connect>=0.6.0",
    # Database ingestion (write support)
    "psycopg[binary]>=3.0.0",
    "mysql-connector-python>=8.0.0",
]

dev = [
    "pytest>=7.0.0",
    "pytest-benchmark>=4.0.0",
    "pytest-cov>=4.1.0",
    "memory-profiler>=0.61.0",
    "psutil>=5.9.0",
    "pytest-xdist>=3.3.0",
    "pytest-timeout>=2.2.0",
    "mock>=4.0.0",
    "ruff>=0.1.0",
    "mypy>=1.0.0",
    # Type stubs for optional dependencies
    "types-chardet>=5.0.0",
    "types-openpyxl>=3.0.0",
    "types-pyyaml>=6.0.0",
    "types-requests>=2.31.0",
    "types-python-dateutil>=2.8.0",
    "types-setuptools>=68.0.0",
    # Note: pandas-stubs and pyarrow-stubs are available but not included
    # as they are large and pandas/pyarrow are already in dev dependencies
    # for testing. Add them if needed for stricter type checking.
    "hypothesis>=6.0.0",
    "pandas>=1.5.0", # Often used in tests or examples (pandas has built-in type stubs)
    # Security tools
    "bandit[toml]>=1.7.5",
    "pip-audit>=2.6.1",
    "safety>=2.3.0",
    # Code quality
    "vulture>=2.10",
    "radon>=6.0.0",
    "pydocstyle[toml]>=6.3.0",
    # Coverage
    "coverage[toml]>=7.3.0",
    # Dependency management
    "pip-tools>=7.3.0",
    "pipdeptree>=2.9.0",
    "pip-licenses>=4.3.0",
    # Build tools
    "build>=1.0.0",
    "twine>=4.0.0",
    "check-wheel-contents>=0.5.0",
]

[project.urls]
Homepage = "https://github.com/datenoio/iterabledata/"
Repository = "https://github.com/datenoio/iterabledata/"

[tool.setuptools.dynamic]
version = {attr = "iterable.__version__"}
readme = {file = ["README.md"], content-type = "text/markdown"}

[tool.setuptools.packages.find]
exclude = ["tests", "tests.*"]

[tool.setuptools.package-data]
iterable = ["py.typed"]

[tool.ruff]
line-length = 120
target-version = "py310"

[tool.ruff.lint]
select = ["E", "F", "I", "B", "UP"]
ignore = []

[tool.pytest.ini_options]
addopts = "--doctest-modules --verbose --cov=iterable --cov-report=term-missing --cov-report=html --cov-report=xml"
testpaths = ["tests"]
norecursedirs = ["examples", ".git", ".venv", "__pycache__", "*.egg", "build", "dist", "node_modules", "docs"]
python_files = ["test_*.py", "*_test.py"]
python_classes = ["Test*"]
python_functions = ["test_*"]
timeout = 300
timeout_method = "thread"
markers = [
    "stress: marks tests as stress tests (creates large files, may take long time)",
    "slow: marks tests as slow running (may take several minutes)",
    "benchmark: marks tests as benchmarks (use --benchmark-only to run)",
]

[tool.coverage.run]
source = ["iterable"]
omit = ["*/tests/*", "*/test_*.py", "*/__pycache__/*"]
branch = true

[tool.coverage.report]
precision = 2
show_missing = true
skip_covered = false
fail_under = 75
exclude_lines = [
    "pragma: no cover",
    "def __repr__",
    "raise AssertionError",
    "raise NotImplementedError",
    "if __name__ == .__main__.:",
    "if TYPE_CHECKING:",
    "@abstractmethod",
]

[tool.coverage.html]
directory = "htmlcov"

[tool.bandit]
exclude_dirs = ["tests", "build", "dist", ".venv", "__pycache__", "examples"]
skips = ["B101"]  # Skip assert_used warnings in tests

[tool.pydocstyle]
convention = "google"
match = "(?!test_).*\\.py"
match-dir = "(?!build|dist|tests|__pycache__)"

[tool.mypy]
python_version = "3.10"
warn_return_any = true
warn_unused_configs = true
disallow_untyped_defs = false  # Set to true after fixing all errors
disallow_incomplete_defs = false  # Set to true after fixing all errors
check_untyped_defs = true
disallow_untyped_decorators = false  # Set to true after fixing all errors
no_implicit_optional = true
warn_redundant_casts = true
warn_unused_ignores = true
warn_no_return = true
warn_unreachable = true
strict_equality = true
show_error_codes = true
show_error_context = true
pretty = true

# Per-module options
[[tool.mypy.overrides]]
module = [
    "tests.*",
    "examples.*",
    "dev.*",
]
disallow_untyped_defs = false
ignore_errors = false

# Ignore missing imports for optional dependencies
# Note: Type stubs are available for many optional dependencies via types-* packages
# (e.g., types-pyyaml, types-requests). These are installed as dev dependencies.
# For packages without type stubs, we use ignore_missing_imports.
[[tool.mypy.overrides]]
module = [
    "iterable.datatypes.*",
    "iterable.codecs.*",
    "iterable.db.*",
    "iterable.engines.*",
]
ignore_missing_imports = true  # Many formats have optional dependencies without type stubs

# Packages with type stubs available (no ignore_missing_imports needed):
# - pandas (pandas-stubs package available, but not installed by default)
# - pyarrow (pyarrow-stubs package available, but not installed by default)
# - pyyaml (types-pyyaml)
# - requests (types-requests)
# - chardet (types-chardet)
# - openpyxl (types-openpyxl)
# - pydantic (built-in type stubs)
# - duckdb (built-in type stubs)
# 
# Packages without type stubs (ignore_missing_imports = true):
# - Most other optional dependencies (pyorc, lxml, pymongo, etc.)