pyproject.toml

[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"

[project]
name = "pyscrappy"
version = "1.6.4"
description = "A robust, all-in-one Python web scraping toolkit"
readme = "README.md"
license = "MIT"
requires-python = ">=3.9"
authors = [
    { name = "Vedaant Singh" },
    { name = "Vedant Tibrewal" },
]
keywords = [
    "scraping", "web-scraping", "crawler", "data-extraction",
    "beautifulsoup", "httpx", "playwright",
]
classifiers = [
    "Development Status :: 4 - Beta",
    "Intended Audience :: Developers",
    "Intended Audience :: Science/Research",
    "License :: OSI Approved :: MIT License",
    "Operating System :: OS Independent",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.9",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Topic :: Internet :: WWW/HTTP",
    "Topic :: Software Development :: Libraries :: Python Modules",
    "Typing :: Typed",
]
dependencies = [
    "httpx>=0.24",
    "beautifulsoup4>=4.12",
    "lxml>=4.9",
]

[project.optional-dependencies]
browser = ["playwright>=1.40"]
dataframe = ["pandas>=1.5"]
# TLS-fingerprint impersonation for ScraperConfig(impersonate=...), to get past
# anti-bot filters that block plain HTTP clients. curl_cffi ships a patched
# libcurl, so it's an opt-in extra rather than a core dependency.
stealth = ["curl_cffi>=0.6"]
# YAML export for ScrapeResult.to_yaml() / save() to a .yaml/.yml file.
yaml = ["PyYAML>=6.0"]
# Parquet export for ScrapeResult.to_parquet() / save() to a .parquet file.
parquet = [
    "pandas>=1.5",
    "pyarrow>=10.0",
]
# Excel export for ScrapeResult.to_excel() / save() to a .xlsx file.
excel = [
    "pandas>=1.5",
    "openpyxl>=3.1",
]
# The MCP SDK requires Python >=3.10. The core library still supports 3.9, so
# gate these deps behind a marker: on 3.9 they're skipped, on 3.10+ installed.
mcp = [
    # FastMCP graduated out of the mcp SDK (removed from mcp 2.x) into the
    # standalone `fastmcp` package, which we import directly. It requires
    # Python >=3.10; the core library still supports 3.9, so gate behind a marker.
    "fastmcp>=2.0; python_version >= '3.10'",
    "anyio>=4.0; python_version >= '3.10'",
]
all = [
    "playwright>=1.40",
    "pandas>=1.5",
    "curl_cffi>=0.6",
    "PyYAML>=6.0",
    "pyarrow>=10.0",
    "openpyxl>=3.1",
    "fastmcp>=2.0; python_version >= '3.10'",
    "anyio>=4.0; python_version >= '3.10'",
]

[project.scripts]
pyscrappy = "pyscrappy.cli:main"
pyscrappy-mcp = "pyscrappy.mcp.server:main"

[project.urls]
Homepage = "https://github.com/mldsveda/PyScrappy"
Repository = "https://github.com/mldsveda/PyScrappy"
Issues = "https://github.com/mldsveda/PyScrappy/issues"

[tool.hatch.build.targets.wheel]
packages = ["src/pyscrappy"]

[tool.ruff]
target-version = "py39"
line-length = 100

[tool.ruff.lint]
select = ["E", "F", "I", "W"]
# The formatter owns line width; E501 only fires on lines it can't split
# (long string literals), so enforcing it adds noise, not value.
ignore = ["E501"]

[tool.pytest.ini_options]
testpaths = ["tests"]
# Integration tests hit the live network and are opt-in. They're excluded from
# the default run (they're slow and can be rate-limited); run them explicitly
# with:  pytest -m integration
markers = [
    "integration: live tests that make real network requests (deselected by default)",
]
addopts = "-m 'not integration'"

[tool.mypy]
python_version = "3.9"
strict = true