pyproject.toml

# Copyright The Lightning AI team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

[build-system]
requires = [
  "setuptools",
  "wheel",
]

[tool.ruff]
target-version = "py38"
line-length = 120
# Exclude a variety of commonly ignored directories.
exclude = [
  ".git",
  "docs",
  "src/litdata/utilities/_pytree.py",
]
# Enable Pyflakes `E` and `F` codes by default.
lint.select = [
  "E",
  "F",      # see: https://pypi.org/project/pyflakes
  "RUF018", # see: https://docs.astral.sh/ruff/rules/assignment-in-assert
  "S",      # see: https://pypi.org/project/flake8-bandit
  "UP",     # see: pyupgrade
  "W",      # see: https://pypi.org/project/pycodestyle
]
lint.extend-select = [
  "C4",     # see: https://pypi.org/project/flake8-comprehensions
  "D",
  "I",      # see: isort
  "NPY201", # see: https://docs.astral.sh/ruff/rules/numpy2-deprecation
  "PT",     # see: https://pypi.org/project/flake8-pytest-style
  "RET",    # see: https://pypi.org/project/flake8-return
  "RUF100", # yesqa
  "SIM",    # see: https://pypi.org/project/flake8-simplify
]
lint.ignore = [
  "E731", # Do not assign a lambda expression, use a def
  "S101", # todo: Use of `assert` detected
]
lint.per-file-ignores."examples/**" = [
  "D100",
  "D101",
  "D102",
  "D103",
  "D104",
  "D105",
  "D107", # Missing docstring in public module, class, method, function, package
  "D205", # todo: 1 blank line required between summary line and description
  "D401",
  "D404", # First line should be in imperative mood; try rephrasing
  "S311", # Standard pseudo-random generators are not suitable for cryptographic purposes
]
lint.per-file-ignores."setup.py" = [ "D100", "SIM115" ]
lint.per-file-ignores."src/**" = [
  "D100", # Missing docstring in public module
  "D101", # todo: Missing docstring in public class
  "D102", # todo: Missing docstring in public method
  "D103", # todo: Missing docstring in public function
  "D104", # Missing docstring in public package
  "D105", # todo: Missing docstring in magic method
  "D107", # todo: Missing docstring in __init__
  "D205", # todo: 1 blank line required between summary line and description
  "D401",
  "D404", # todo: First line should be in imperative mood; try rephrasing
  "S310", # todo: Audit URL open for permitted schemes. Allowing use of `file:` or custom schemes is often unexpected.
  "S602", # todo: `subprocess` call with `shell=True` identified, security issue
  "S605", # todo: Starting a process with a shell: seems safe, but may be changed in the future; consider rewriting without `shell`
  "S607", # todo: Starting a process with a partial executable path
]
lint.per-file-ignores."tests/**" = [
  "D100",
  "D101",
  "D102",
  "D103",
  "D104",
  "D105",
  "D107", # Missing docstring in public module, class, method, function, package
  "D401",
  "D404", # First line should be in imperative mood; try rephrasing
  "S105",
  "S106", # todo: Possible hardcoded password: ...
]
# Unlike Flake8, default to a complexity level of 10.
lint.mccabe.max-complexity = 10
# Use Google-style docstrings.
lint.pydocstyle.convention = "google"

[tool.codespell]
# Todo: enable also python files in a next step
#skip = '*.py'
quiet-level = 3
# comma separated list of words; waiting for:
#  https://github.com/codespell-project/codespell/issues/2839#issuecomment-1731601603
# also adding links until they ignored by its: nature
#  https://github.com/codespell-project/codespell/issues/2243#issuecomment-1732019960
ignore-words-list = "cancelation"

[tool.docformatter]
recursive = true
# this need to be shorter as some docstings are r"""...
wrap-summaries = 119
wrap-descriptions = 120
blank = true

[tool.pytest.ini_options]
testpaths = [
  "tests",
]
norecursedirs = [
  ".git",
  ".github",
  "dist",
  "build",
  "docs",
]
addopts = [
  "--strict-markers",
  "--doctest-modules",
  "--color=yes",
  "--disable-pytest-warnings",
  "--ignore=legacy/checkpoints",
]
markers = [
  "cloud: Run the cloud tests for example",
]
filterwarnings = [
  "error::FutureWarning",
]
xfail_strict = true
junit_duration_report = "call"

[tool.coverage.report]
exclude_lines = [
  "pragma: no cover",
  "warnings",
  "pass",
  "rank_zero_warn",
  "raise NotImplementedError",
]

[tool.mypy]
files = [
  "src",
]
# This section is for folders with "-" as they are not valid python modules
exclude = [
  "src/litdata/utilities/_pytree.py",
  "src/litdata/streaming/item_loader.py",
  "src/litdata/utilities/breakpoint.py",
  "src/litdata/utilities/subsample.py",
  "src/litdata/streaming/sampler.py",
  "src/litdata/streaming/shuffle.py",
  "src/litdata/imports.py",
  "src/litdata/imports.py",
  "src/litdata/processing/data_processor.py",
]
install_types = "True"
non_interactive = "True"
disallow_untyped_defs = "True"
ignore_missing_imports = "True"
show_error_codes = "True"
warn_redundant_casts = "True"
warn_unused_configs = "True"
warn_unused_ignores = "True"
allow_redefinition = "True"
# disable this rule as the Trainer attributes are defined in the connectors, not in its __init__
disable_error_code = "attr-defined"
# style choices
warn_no_return = "False"

# Ignore mypy errors for these files
# TODO: the goal is for this to be empty
[[tool.mypy.overrides]]
# the list can be generated with:
# mypy --no-error-summary 2>&1 | tr ':' ' ' | awk '{print $1}' | sort | uniq | sed 's/\.py//g; s|src/||g;  s|\/|\.|g' | xargs -I {} echo '"{}",'
module = [
]
ignore_errors = "True"