From a701ba0df609a99b8a7e0d55102743a3d4ea7539 Mon Sep 17 00:00:00 2001 From: Janos Gabler Date: Wed, 9 Mar 2022 17:49:14 +0100 Subject: [PATCH] First release of dags. (#1) --- .github/ISSUE_TEMPLATE/bug-report.md | 29 ++ .github/ISSUE_TEMPLATE/enhancement.md | 25 ++ .github/ISSUE_TEMPLATE/feature_request.md | 21 ++ .../pull_request_template.md | 11 + .github/workflows/main.yml | 67 ++++ .github/workflows/publish-to-pypi.yml | 34 ++ .gitignore | 132 +++++++ .pre-commit-config.yaml | 110 ++++++ .readthedocs.yml | 10 + CHANGES.rst | 14 + LICENSE | 2 +- MANIFEST.in | 11 + README.rst | 60 ++++ codecov.yml | 19 ++ docs/Makefile | 20 ++ docs/make.bat | 36 ++ docs/rtd_environment.yml | 22 ++ docs/source/_static/css/custom.css | 182 ++++++++++ docs/source/api.rst | 10 + docs/source/conf.py | 140 ++++++++ docs/source/index.rst | 9 + environment.yml | 32 ++ pyproject.toml | 23 ++ setup.cfg | 41 +++ src/dags/__init__.py | 10 + src/dags/dag.py | 321 ++++++++++++++++++ tests/test_dag.py | 122 +++++++ tox.ini | 63 ++++ 28 files changed, 1575 insertions(+), 1 deletion(-) create mode 100644 .github/ISSUE_TEMPLATE/bug-report.md create mode 100644 .github/ISSUE_TEMPLATE/enhancement.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/PULL_REQUEST_TEMPLATE/pull_request_template.md create mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/publish-to-pypi.yml create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 .readthedocs.yml create mode 100644 CHANGES.rst create mode 100644 MANIFEST.in create mode 100644 README.rst create mode 100644 codecov.yml create mode 100644 docs/Makefile create mode 100644 docs/make.bat create mode 100644 docs/rtd_environment.yml create mode 100644 docs/source/_static/css/custom.css create mode 100644 docs/source/api.rst create mode 100644 docs/source/conf.py create mode 100644 docs/source/index.rst create mode 100644 environment.yml create mode 100644 pyproject.toml create mode 100644 setup.cfg create mode 100644 src/dags/__init__.py create mode 100644 src/dags/dag.py create mode 100644 tests/test_dag.py create mode 100644 tox.ini diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md new file mode 100644 index 0000000..10332f4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -0,0 +1,29 @@ +--- +name: Bug Report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +### Bug description + +A clear and concise description of what the bug is. + +### To Reproduce + +Ideally, provide a minimal code example. If that's not possible, describe steps to reproduce the bug. + +### Expected behavior + +A clear and concise description of what you expected to happen. + +### Screenshots/Error messages + +If applicable, add screenshots to help explain your problem. + +### System + + - OS: [e.g. Ubuntu 18.04] + - Version [e.g. 0.0.1] diff --git a/.github/ISSUE_TEMPLATE/enhancement.md b/.github/ISSUE_TEMPLATE/enhancement.md new file mode 100644 index 0000000..1654f28 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/enhancement.md @@ -0,0 +1,25 @@ +--- +name: Enhancement +about: Enhance an existing component. +title: '' +labels: enhancement +assignees: '' + +--- + +* dags version used, if any: +* Python version, if any: +* Operating System: + +### What would you like to enhance and why? Is it related to an issue/problem? + +A clear and concise description of the current implementation and its limitations. + +### Describe the solution you'd like + +A clear and concise description of what you want to happen. + +### Describe alternatives you've considered + +A clear and concise description of any alternative solutions or features you've +considered and why you have discarded them. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..cf66450 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,21 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: feature-request +assignees: '' + +--- + +### Current situation + +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]; Currently there is no way of [...] + +### Desired Situation + +What functionality should become possible or easier? + +### Proposed implementation + +How would you implement the new feature? Did you consider alternative implementations? +You can start by describing interface changes like a new argument or a new function. There is no need to get too detailed here. diff --git a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md new file mode 100644 index 0000000..5a1fce7 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md @@ -0,0 +1,11 @@ +### What problem do you want to solve? + +Reference the issue or discussion, if there is any. Provide a description of your +proposed solution. + +### Todo + +- [ ] Target the right branch and pick an appropriate title. +- [ ] Put `Closes #XXXX` in the first PR comment to auto-close the relevant issue once + the PR is accepted. This is not applicable if there is no corresponding issue. +- [ ] Any steps that still need to be done. diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..488da2c --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,67 @@ +name: main +on: + push: + branches: + - main + pull_request: + branches: + - '*' + +# Automatically cancel a previous run. +concurrency: + group: ${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + + run-tests: + + name: Run tests for ${{ matrix.os }} on ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] + python-version: ['3.7', '3.8', '3.9', '3.10'] + + steps: + - uses: actions/checkout@v2 + - uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: ${{ matrix.python-version }} + + - name: Install core dependencies. + shell: bash -l {0} + run: conda install -c conda-forge tox-conda + + - name: Run pytest. + shell: bash -l {0} + run: tox -e pytest -- -m "not slow" --cov-report=xml --cov=./ + + - name: Upload coverage report. + if: runner.os == 'Linux' && matrix.python-version == '3.9' + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + + docs: + + name: Run documentation. + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: 3.9 + + - name: Install core dependencies. + shell: bash -l {0} + run: conda install -c conda-forge tox-conda + + - name: Build docs + shell: bash -l {0} + run: tox -e sphinx diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml new file mode 100644 index 0000000..710ebb1 --- /dev/null +++ b/.github/workflows/publish-to-pypi.yml @@ -0,0 +1,34 @@ +name: PyPI + +on: push + +jobs: + build-n-publish: + name: Build and publish Python 🐍 distributions 📦 to PyPI + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install pypa/build + run: >- + python -m + pip install + build + --user + - name: Build a binary wheel and a source tarball + run: >- + python -m + build + --sdist + --wheel + --outdir dist/ + - name: Publish distribution 📦 to PyPI + if: startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@master + with: + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..db3addb --- /dev/null +++ b/.gitignore @@ -0,0 +1,132 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + + +src/dags/_version.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..c31aae7 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,110 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.1.0 + hooks: + - id: check-merge-conflict + - id: debug-statements + - id: end-of-file-fixer +- repo: https://github.com/asottile/reorder_python_imports + rev: v2.7.1 + hooks: + - id: reorder-python-imports + types: [python] +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.1.0 + hooks: + - id: check-added-large-files + args: ['--maxkb=100'] + - id: check-case-conflict + - id: check-merge-conflict + - id: check-vcs-permalinks + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + - id: fix-byte-order-marker + - id: mixed-line-ending + - id: no-commit-to-branch + args: [--branch, main] + - id: trailing-whitespace +- repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.9.0 + hooks: + - id: python-check-blanket-noqa + - id: python-check-mock-methods + - id: python-no-eval + - id: python-no-log-warn + - id: python-use-type-annotations + - id: rst-backticks + - id: rst-directive-colons + - id: rst-inline-touching-normal + - id: text-unicode-replacement-char +- repo: https://github.com/asottile/blacken-docs + rev: v1.12.1 + hooks: + - id: blacken-docs + additional_dependencies: [black] + types: [rst] +- repo: https://github.com/psf/black + rev: 22.1.0 + hooks: + - id: black + types: [python] +- repo: https://github.com/PyCQA/flake8 + rev: 4.0.1 + hooks: + - id: flake8 + types: [python] + additional_dependencies: [ + flake8-alfred, + flake8-bugbear, + flake8-builtins, + flake8-comprehensions, + flake8-docstrings, + flake8-eradicate, + flake8-print, + flake8-pytest-style, + flake8-todo, + flake8-typing-imports, + flake8-unused-arguments, + pep8-naming, + pydocstyle, + Pygments, + ] +- repo: https://github.com/PyCQA/doc8 + rev: 0.10.1 + hooks: + - id: doc8 +- repo: meta + hooks: + - id: check-hooks-apply + - id: check-useless-excludes + # - id: identity # Prints all files passed to pre-commits. Debugging. +- repo: https://github.com/mgedmin/check-manifest + rev: "0.47" + hooks: + - id: check-manifest + args: [--no-build-isolation] + additional_dependencies: [setuptools-scm, toml] +- repo: https://github.com/PyCQA/doc8 + rev: 0.10.1 + hooks: + - id: doc8 +- repo: https://github.com/asottile/setup-cfg-fmt + rev: v1.20.0 + hooks: + - id: setup-cfg-fmt +- repo: https://github.com/econchick/interrogate + rev: 1.5.0 + hooks: + - id: interrogate + args: [-v, --fail-under=20] + exclude: ^(tests|docs|setup\.py) +- repo: https://github.com/codespell-project/codespell + rev: v2.1.0 + hooks: + - id: codespell +- repo: https://github.com/asottile/pyupgrade + rev: v2.31.0 + hooks: + - id: pyupgrade + args: [--py37-plus] diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..5e35e9f --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,10 @@ +version: 2 + +build: + image: latest + +python: + version: 3.8 + +conda: + environment: docs/rtd_environment.yml diff --git a/CHANGES.rst b/CHANGES.rst new file mode 100644 index 0000000..ee33297 --- /dev/null +++ b/CHANGES.rst @@ -0,0 +1,14 @@ +Changes +======= + +This is a record of all past dags releases and what went into them in reverse +chronological order. We follow `semantic versioning `_ and all +releases are available on `Anaconda.org +`_. + + + +0.1.0 - 2022-03-08 +------------------ + +- :gh:`1` releases the initial version of dags. diff --git a/LICENSE b/LICENSE index 1fbf333..e746cf5 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022 Open Source Economics +Copyright (c) 2022 Janoś Gabler and Tobias Raabe Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..5b8f8a0 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,11 @@ +include CITATION +include LICENSE +include README.rst +include CHANGES.rst + +exclude *.yaml +exclude *.yml +exclude tox.ini + +prune docs +prune tests diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..c5f8594 --- /dev/null +++ b/README.rst @@ -0,0 +1,60 @@ +dags +==== + +.. start-badges + +.. image:: https://img.shields.io/pypi/v/dags?color=blue + :alt: PyPI + :target: https://pypi.org/project/dags + +.. image:: https://img.shields.io/pypi/pyversions/dags + :alt: PyPI - Python Version + :target: https://pypi.org/project/dags + +.. image:: https://img.shields.io/conda/vn/conda-forge/dags.svg + :target: https://anaconda.org/conda-forge/dags + +.. image:: https://img.shields.io/conda/pn/conda-forge/dags.svg + :target: https://anaconda.org/conda-forge/dags + +.. image:: https://img.shields.io/pypi/l/dags + :alt: PyPI - License + :target: https://pypi.org/project/dags + +.. image:: https://readthedocs.org/projects/dags/badge/?version=latest + :target: https://dags.readthedocs.io/en/latest + +.. image:: https://img.shields.io/github/workflow/status/OpenSourceEconomics/dags/main/main + :target: https://github.com/OpenSourceEconomics/dags/actions?query=branch%3Amain + +.. image:: https://codecov.io/gh/OpenSourceEconomics/dags/branch/main/graph/badge.svg + :target: https://codecov.io/gh/OpenSourceEconomics/dags + +.. image:: https://results.pre-commit.ci/badge/github/OpenSourceEconomics/dags/main.svg + :target: https://results.pre-commit.ci/latest/github/OpenSourceEconomics/dags/main + :alt: pre-commit.ci status + +.. image:: https://img.shields.io/badge/code%20style-black-000000.svg + :target: https://github.com/psf/black + +.. end-badges + +Installation +------------ + +dags is available on `PyPI `_ and `Anaconda.org +`_. Install it with + +.. code-block:: console + + $ pip install dags + + # or + + $ conda install -c conda-forge dags + + +About +----- + +dags provides Tools to create executable dags from interdependent functions. diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..13d1231 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,19 @@ +codecov: + notify: + require_ci_to_pass: yes + +coverage: + precision: 2 + round: down + range: "50...100" + status: + patch: + default: + target: 80% + project: + default: + target: 80% + +ignore: + - ".tox/**/*" + - "setup.py" diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..b031444 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = dags +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..2b963f8 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build +set SPHINXPROJ=dags + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/docs/rtd_environment.yml b/docs/rtd_environment.yml new file mode 100644 index 0000000..ba8543c --- /dev/null +++ b/docs/rtd_environment.yml @@ -0,0 +1,22 @@ +channels: + - conda-forge + - nodefaults + +dependencies: + - python=3.9 + - pip + - setuptools_scm + - toml + + # Package + - networkx + + # Documentation + - sphinx + - sphinx-autoapi + - sphinx-copybutton + - sphinx-panels + - pydata-sphinx-theme>=0.3.0 + + - pip: + - ../ diff --git a/docs/source/_static/css/custom.css b/docs/source/_static/css/custom.css new file mode 100644 index 0000000..36b3df0 --- /dev/null +++ b/docs/source/_static/css/custom.css @@ -0,0 +1,182 @@ +/* Remove execution count for notebook cells. */ +div.prompt { + display: none; +} + +/* Getting started index page */ + +.intro-card { + background: #fff; + border-radius: 0; + padding: 30px 10px 10px 10px; + margin: 10px 0px; + max-height: 85%; +} + +.intro-card .card-text { + margin: 20px 0px; +} + +div#index-container { + padding-bottom: 20px; +} + +a#index-link { + color: #333; + text-decoration: none; +} + +/* reference to user guide */ +.gs-torefguide { + align-items: center; + font-size: 0.9rem; +} + +.gs-torefguide .badge { + background-color: #130654; + margin: 10px 10px 10px 0px; + padding: 5px; +} + +.gs-torefguide a { + margin-left: 5px; + color: #130654; + border-bottom: 1px solid #FFCA00f3; + box-shadow: 0px -10px 0px #FFCA00f3 inset; +} + +.gs-torefguide p { + margin-top: 1rem; +} + +.gs-torefguide a:hover { + margin-left: 5px; + color: grey; + text-decoration: none; + border-bottom: 1px solid #b2ff80f3; + box-shadow: 0px -10px 0px #b2ff80f3 inset; +} + +/* selecting constraints guide */ +.intro-card { + background:#FFF; + border-radius:0; + padding: 30px 10px 10px 10px; + margin: 10px 0px; +} + +.intro-card .card-text { + margin:20px 0px; + /*min-height: 150px; */ +} + +.intro-card .card-img-top { + margin: 10px; +} + +.install-block { + padding-bottom: 30px; +} + +.install-card .card-header { + border: none; + background-color:white; + color: #150458; + font-size: 1.1rem; + font-weight: bold; + padding: 1rem 1rem 0rem 1rem; +} + +.install-card .card-footer { + border: none; + background-color:white; +} + +.install-card pre { + margin: 0 1em 1em 1em; +} + +.custom-button { + background-color:#DCDCDC; + border: none; + color: #484848; + text-align: center; + text-decoration: none; + display: inline-block; + font-size: 0.9rem; + border-radius: 0.5rem; + max-width: 120px; + padding: 0.5rem 0rem; +} + +.custom-button a { + color: #484848; +} + +.custom-button p { + margin-top: 0; + margin-bottom: 0rem; + color: #484848; +} + +/* selecting constraints guide collapsed cards */ + +.tutorial-accordion { + margin-top: 20px; + margin-bottom: 20px; +} + +.tutorial-card .card-header.card-link .btn { + margin-right: 12px; +} + +.tutorial-card .card-header.card-link .btn:after { + content: "-"; +} + +.tutorial-card .card-header.card-link.collapsed .btn:after { + content: "+"; +} + +.tutorial-card-header-1 { + justify-content: space-between; + align-items: center; +} + +.tutorial-card-header-2 { + justify-content: flex-start; + align-items: center; + font-size: 1.3rem; +} + +.tutorial-card .card-header { + cursor: pointer; + background-color: white; +} + +.tutorial-card .card-body { + background-color: #F0F0F0; +} + +.tutorial-card .badge:hover { + background-color: grey; +} + +/* tables in selecting constraints guide */ + +table.rows th { + background-color: #F0F0F0; + border-style: solid solid solid solid; + border-width: 0px 0px 0px 0px; + border-color: #F0F0F0; + text-align: center; +} + +table.rows tr:nth-child(even) { + background-color: #F0F0F0; + text-align: right; +} +table.rows tr:nth-child(odd) { + background-color: #FFFFFF; + text-align: right; +} diff --git a/docs/source/api.rst b/docs/source/api.rst new file mode 100644 index 0000000..6a2b7ea --- /dev/null +++ b/docs/source/api.rst @@ -0,0 +1,10 @@ +API Reference +============= + +The following documents are auto-generated and not carefully edited. They provide direct +access to the source code and the docstrings. + +.. toctree:: + :titlesonly: + + /autoapi/dags/index diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..3667002 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,140 @@ +import os +from importlib.metadata import version + + +author = "Janoś Gabler, Tobias Raabe" + +# Set variable so that todos are shown in local build +on_rtd = os.environ.get("READTHEDOCS") == "True" + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.todo", + "sphinx.ext.coverage", + "sphinx.ext.extlinks", + "sphinx.ext.intersphinx", + "sphinx.ext.mathjax", + "sphinx.ext.viewcode", + "sphinx.ext.napoleon", + "sphinx_panels", + "autoapi.extension", +] + +autodoc_member_order = "bysource" + +autodoc_mock_imports = [ + "pandas", + "pytest", + "numpy", + "jax", +] + +extlinks = { + "ghuser": ("https://github.com/%s", "@"), + "gh": ("https://github.com/OpenSourceEconomics/dags/pulls/%s", "#"), +} + +intersphinx_mapping = { + "numpy": ("https://numpy.org/doc/stable", None), + "np": ("https://numpy.org/doc/stable", None), + "pandas": ("https://pandas.pydata.org/pandas-docs/stable", None), + "pd": ("https://pandas.pydata.org/pandas-docs/stable", None), + "python": ("https://docs.python.org/3.9", None), +} + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] +html_static_path = ["_static"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +source_suffix = ".rst" + +# The master toctree document. +master_doc = "index" + +# General information about the project. +project = "dags" +copyright = f"2022, {author}" # noqa: A001 + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. + +# The version, including alpha/beta/rc tags, but not commit hash and datestamps +release = version("dags") +# The short X.Y version. +version = ".".join(release.split(".")[:2]) + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. + +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "**.ipynb_checkpoints"] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + +# If true, `todo` and `todoList` produce output, else they produce nothing. +if on_rtd: + pass +else: + todo_include_todos = True + todo_emit_warnings = True + +# Remove prefixed $ for bash, >>> for Python prompts, and In [1]: for IPython prompts. +copybutton_prompt_text = r"\$ |>>> |In \[\d\]: " +copybutton_prompt_is_regexp = True + +# Configuration for autoapi +autoapi_type = "python" +autoapi_dirs = ["../../src"] +autoapi_keep_files = False +autoapi_add_toctree_entry = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = "pydata_sphinx_theme" + +# html_logo = "_static/images/logo.svg" + +html_theme_options = { + "github_url": "https://github.com/OpenSourceEconomics/dags", +} + +html_css_files = ["css/custom.css"] + + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ["_static"] # noqa: E800 + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. + +# This is required for the alabaster theme +# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars +html_sidebars = { + "**": [ + "relations.html", # needs 'show_related': True theme option to display + "searchbox.html", + ] +} diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..549fce2 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,9 @@ +dags +==== + +``dags`` contains tools to create executable dags from interdependent functions. + +.. toctree:: + :maxdepth: 1 + + api diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..e241376 --- /dev/null +++ b/environment.yml @@ -0,0 +1,32 @@ +name: dags + +channels: + - conda-forge + - nodefaults + +dependencies: + - python >=3.9 + - pip + - setuptools_scm + - toml + + # Package + - networkx + + # Testing + - pre-commit + - pytest + - pytest-cov + - pytest-xdist + - tox-conda + + # Documentation + - sphinx >=4 + - sphinx-autoapi + - sphinx-copybutton + - sphinx-panels + - pydata-sphinx-theme>=0.3.0 + + # Development + - jupyterlab + - nbsphinx diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a92c292 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,23 @@ +[build-system] +requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.0"] +build-backend = "setuptools.build_meta" + + +[tool.setuptools_scm] +write_to = "src/dags/_version.py" + + +[tool.nbqa.config] +isort = "setup.cfg" +black = "pyproject.toml" + + +[tool.nbqa.mutate] +isort = 1 +black = 1 +pyupgrade = 1 + + +[tool.nbqa.addopts] +isort = ["--treat-comment-as-code", "# %%", "--profile=black"] +pyupgrade = ["--py37-plus"] diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..47c3fe6 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,41 @@ +[metadata] +name = dags +description = Tools to create executable dags from interdependent functions. +long_description = file: README.rst +long_description_content_type = text/x-rst +url = https://github.com/OpenSourceEconomics/dags +author = Janoś Gabler, Tobias Raabe +author_email = janos.gabler@gmail.com +license = MIT +license_file = LICENSE +platforms = unix, linux, osx, cygwin, win32 +classifiers = + Development Status :: 3 - Alpha + License :: OSI Approved :: MIT License + Operating System :: MacOS :: MacOS X + Operating System :: Microsoft :: Windows + Operating System :: POSIX + Programming Language :: Python :: 3 + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Topic :: Utilities + +[options] +packages = find: +install_requires = + networkx +python_requires = >=3.7 +include_package_data = True +package_dir = + =src +zip_safe = False + +[options.packages.find] +where = src + +[check-manifest] +ignore = + src/dags/_version.py diff --git a/src/dags/__init__.py b/src/dags/__init__.py new file mode 100644 index 0000000..d70e31e --- /dev/null +++ b/src/dags/__init__.py @@ -0,0 +1,10 @@ +from dags.dag import aggregate_functions +from dags.dag import concatenate_functions +from dags.dag import get_ancestors + + +__all__ = [ + "concatenate_functions", + "aggregate_functions", + "get_ancestors", +] diff --git a/src/dags/dag.py b/src/dags/dag.py new file mode 100644 index 0000000..dacca5b --- /dev/null +++ b/src/dags/dag.py @@ -0,0 +1,321 @@ +import functools +import inspect +import textwrap + +import networkx as nx + + +def concatenate_functions(functions, targets, return_dict: bool = False): + """Combine functions to one function that generates the targets. + + Functions can depend on the output of other functions as inputs, as long as the + dependencies can be described by a directed acyclic graph (DAG). + + Functions that are not required to produce the target will simply be ignored. + + The arguments of the combined function are all arguments of relevant functions + that are not themselves function names. + + Args: + functions (dict or list): Dict or list of functions. If a list, the function + name is inferred from the __name__ attribute of the entries. If a dict, + the name of the function is set to the dictionary key. + targets (str): Name of the function that produces the target or list of such + function names. + return_dict (bool, optional): Whether the function should return a dictionary + with node names as keys or just the values as a tuple for multiple outputs. + + Returns: + function: A function that produces target when called with suitable arguments. + + """ + functions, targets, single_target = _check_and_process_inputs(functions, targets) + raw_dag = _create_complete_dag(functions) + dag = _limit_dag_to_targets_and_their_ancestors(raw_dag, targets) + signature = _get_signature(functions, dag) + exec_info = _create_execution_info(functions, dag) + if single_target: + concatenated = _create_concatenated_function_single_target(exec_info, signature) + else: + concatenated = _create_concatenated_function_multi_target( + exec_info, signature, targets, return_dict + ) + return concatenated + + +def aggregate_functions(functions, targets, aggregator=lambda a, b: a and b): + """Aggregate the result of targets + + Functions can depend on the output of other functions as inputs, as long as the + dependencies can be described by a directed acyclic graph (DAG). + + All functions in targets need to return a scalar bool. + + Functions that are not required to produce the target will simply be ignored. + + The arguments of the combined function are all arguments of relevant functions + that are not themselves function names. + + Args: + functions (dict or list): Dict or list of functions. If a list, the function + name is inferred from the __name__ attribute of the entries. If a dict, + the name of the function is set to the dictionary key. + targets (str): Name of the function that produces the target or list of such + function names. + + Returns: + function: A function that produces target when called with suitable arguments. + + """ + concatenated = concatenate_functions(functions, targets, False) + aggregated = _add_aggregation(concatenated, aggregator) + return aggregated + + +def get_ancestors(functions, targets, include_target=False): + """Build a DAG and extract all ancestors of target. + + Args: + functions (dict or list): Dict or list of functions. If a list, the function + name is inferred from the __name__ attribute of the entries. If a dict, + with node names as keys or just the values as a tuple for multiple outputs. + targets (str): Name of the function that produces the target function. + include_target (bool): Whether to include the target as its own ancestor. + + Returns: + set: The ancestors + + """ + functions, targets, _ = _check_and_process_inputs(functions, targets) + raw_dag = _create_complete_dag(functions) + dag = _limit_dag_to_targets_and_their_ancestors(raw_dag, targets) + + ancestors = set() + for target in targets: + ancestors = ancestors.union(nx.ancestors(dag, target)) + if include_target: + ancestors.add(target) + return ancestors + + +def _check_and_process_inputs(functions, targets): + if isinstance(functions, (list, tuple)): + functions = {func.__name__: func for func in functions} + + single_target = isinstance(targets, str) + if single_target: + targets = [targets] + + not_strings = [target for target in targets if not isinstance(target, str)] + if not_strings: + raise ValueError( + f"Targets must be strings. The following targets are not: {not_strings}" + ) + + # to-do: add typo suggestions via fuzzywuzzy, see estimagic + targets_not_in_functions = set(targets) - set(functions) + if targets_not_in_functions: + formatted = _format_list_linewise(targets_not_in_functions) + raise ValueError( + f"The following targets have no corresponding function:\n{formatted}" + ) + + return functions, targets, single_target + + +def _create_complete_dag(functions): + """Create the complete DAG. + + This DAG is constructed from all functions and not pruned by specified root nodes or + targets. + + Args: + functions (dict): Dictionary containing functions to build the DAG. + + Returns: + networkx.DiGraph: The complete DAG + + """ + functions_arguments_dict = { + name: list(inspect.signature(function).parameters) + for name, function in functions.items() + } + dag = nx.DiGraph(functions_arguments_dict).reverse() + + return dag + + +def _limit_dag_to_targets_and_their_ancestors(dag, targets): + """Limit DAG to targets and their ancestors. + + Args: + dag (networkx.DiGraph): The complete DAG. + targets (str): Variable of interest. + + Returns: + networkx.DiGraph: The pruned DAG. + + """ + used_nodes = set(targets) + for target in targets: + used_nodes = used_nodes | set(nx.ancestors(dag, target)) + + all_nodes = set(dag.nodes) + + unused_nodes = all_nodes - used_nodes + + dag.remove_nodes_from(unused_nodes) + + return dag + + +def _get_signature(functions, dag): + """Create the signature of the concatenated function. + + Args: + functions (dict): Dictionary containing functions to build the DAG. + dag (networkx.DiGraph): The complete DAG. + + Returns: + inspect.Signature: The signature of the concatenated function. + + """ + function_names = set(functions) + all_nodes = set(dag.nodes) + arguments = sorted(all_nodes - function_names) + + parameter_objects = [] + for arg in arguments: + parameter_objects.append( + inspect.Parameter(name=arg, kind=inspect.Parameter.POSITIONAL_OR_KEYWORD) + ) + + sig = inspect.Signature(parameters=parameter_objects) + return sig + + +def _create_execution_info(functions, dag): + """Create a dictionary with all information needed to execute relevant functions. + + Args: + functions (dict): Dictionary containing functions to build the DAG. + dag (networkx.DiGraph): The complete DAG. + + Returns: + dict: Dictionary with functions and their arguments for each node in the dag. + The functions are already in topological_sort order. + + """ + out = {} + for node in nx.topological_sort(dag): + if node in functions: + info = {} + info["func"] = functions[node] + info["arguments"] = list(inspect.signature(functions[node]).parameters) + + out[node] = info + return out + + +def _create_concatenated_function_single_target(execution_info, signature): + """Create a concatenated function object with correct signature. + + Args: + execution_info (dict): Dictionary with functions and their arguments for each + node in the dag. The functions are already in topological_sort order. + signature (inspect.Signature)): The signature of the concatenated function. + + Returns: + callable: The concatenated function + + """ + parameters = sorted(signature.parameters) + + def concatenated(*args, **kwargs): + results = {**dict(zip(parameters, args)), **kwargs} + for name, info in execution_info.items(): + arguments = _dict_subset(results, info["arguments"]) + result = info["func"](**arguments) + results[name] = result + return result + + concatenated.__signature__ = signature + + return concatenated + + +def _create_concatenated_function_multi_target( + execution_info, + signature, + targets, + return_dict, +): + """Create a concatenated function object with correct signature. + + Args: + execution_info (dict): Dictionary with functions and their arguments for each + node in the dag. The functions are already in topological_sort order. + signature (inspect.Signature)): The signature of the concatenated function. + targets (list): List that is used to determine what is returned and the + order of the outputs. + return_dict (bool): Whether the function should return a dictionary + with node names as keys or just the values as a tuple for multiple outputs. + + Returns: + callable: The concatenated function + + """ + parameters = sorted(signature.parameters) + + def concatenated(*args, **kwargs): + results = {**dict(zip(parameters, args)), **kwargs} + for name, info in execution_info.items(): + arguments = _dict_subset(results, info["arguments"]) + result = info["func"](**arguments) + results[name] = result + + out = {target: results[target] for target in targets} + return out + + concatenated.__signature__ = signature + + if not return_dict: + concatenated = _convert_dict_output_to_tuple(concatenated) + + return concatenated + + +def _add_aggregation(func, aggregator): + @functools.wraps(func) + def with_aggregation(*args, **kwargs): + to_aggregate = func(*args, **kwargs) + agg = to_aggregate[0] + for entry in to_aggregate[1:]: + agg = aggregator(agg, entry) + return agg + + return with_aggregation + + +def _dict_subset(dictionary, keys): + """Reduce dictionary to keys.""" + return {k: dictionary[k] for k in keys} + + +def _convert_dict_output_to_tuple(func): + @functools.wraps(func) + def wrapped(*args, **kwargs): + return tuple(func(*args, **kwargs).values()) + + return wrapped + + +def _format_list_linewise(list_): + formatted_list = '",\n "'.join(list_) + return textwrap.dedent( + """ + [ + "{formatted_list}", + ] + """ + ).format(formatted_list=formatted_list) diff --git a/tests/test_dag.py b/tests/test_dag.py new file mode 100644 index 0000000..7a9e651 --- /dev/null +++ b/tests/test_dag.py @@ -0,0 +1,122 @@ +import inspect + +import pytest +from dags.dag import aggregate_functions +from dags.dag import concatenate_functions +from dags.dag import get_ancestors + + +def _utility(_consumption, _leisure): + return _consumption + _leisure + + +def _leisure(working): + return 24 - working + + +def _consumption(working, wage): + return wage * working + + +def _unrelated(working): # noqa: U100 + raise NotImplementedError() + + +def _complete_utility(wage, working): + """The function that we try to generate dynamically.""" + leis = _leisure(working) + cons = _consumption(working, wage) + util = leis + cons + return util + + +def test_concatenate_functions_single_target(): + concatenated = concatenate_functions( + functions=[_utility, _unrelated, _leisure, _consumption], + targets="_utility", + ) + + calculated_res = concatenated(wage=5, working=8, bla=15) + + expected_res = _complete_utility(wage=5, working=8) + assert calculated_res == expected_res + + calculated_args = set(inspect.signature(concatenated).parameters) + expected_args = {"wage", "working"} + + assert calculated_args == expected_args + + +@pytest.mark.parametrize("return_dict", [True, False]) +def test_concatenate_functions_multi_target(return_dict): + concatenated = concatenate_functions( + functions=[_utility, _unrelated, _leisure, _consumption], + targets=["_utility", "_consumption"], + return_dict=return_dict, + ) + + calculated_res = concatenated(wage=5, working=8) + + expected_res = { + "_utility": _complete_utility(wage=5, working=8), + "_consumption": _consumption(wage=5, working=8), + } + if not return_dict: + expected_res = tuple(expected_res.values()) + assert calculated_res == expected_res + + calculated_args = set(inspect.signature(concatenated).parameters) + expected_args = {"wage", "working"} + + assert calculated_args == expected_args + + +def test_get_ancestors_many_ancestors(): + calculated = get_ancestors( + functions=[_utility, _unrelated, _leisure, _consumption], + targets="_utility", + ) + expected = {"_consumption", "_leisure", "working", "wage"} + + assert calculated == expected + + +def test_get_ancestors_few_ancestors(): + calculated = get_ancestors( + functions=[_utility, _unrelated, _leisure, _consumption], + targets="_unrelated", + ) + + expected = {"working"} + + assert calculated == expected + + +def test_get_ancestors_multiple_targets(): + calculated = get_ancestors( + functions=[_utility, _unrelated, _leisure, _consumption], + targets=["_unrelated", "_consumption"], + ) + + expected = {"wage", "working"} + assert calculated == expected + + +def test_aggregate_functions_with_and(): + funcs = {"f1": lambda: True, "f2": lambda: False} + aggregated = aggregate_functions( + functions=funcs, + targets=["f1", "f2"], + aggregator=lambda a, b: a and b, + ) + assert not aggregated() + + +def test_aggregate_functions_with_or(): + funcs = {"f1": lambda: True, "f2": lambda: False} + aggregated = aggregate_functions( + functions=funcs, + targets=["f1", "f2"], + aggregator=lambda a, b: a or b, + ) + assert aggregated() diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..5111c3d --- /dev/null +++ b/tox.ini @@ -0,0 +1,63 @@ +[tox] +envlist = pytest, sphinx +skipsdist = True +skip_missing_interpreters = True + +[testenv] +basepython = python + +[testenv:pytest] +setenv = + CONDA_DLL_SEARCH_MODIFICATION_ENABLE = 1 +conda_channels = + conda-forge + nodefaults +conda_deps = + networkx + pytest + pytest-cov + pytest-mock + pytest-xdist +commands = pytest {posargs} + +[testenv:sphinx] +changedir = docs/source +conda_env = docs/rtd_environment.yml +commands = + sphinx-build -T -b html -d {envtmpdir}/doctrees . {envtmpdir}/html + - sphinx-build -T -b linkcheck -d {envtmpdir}/doctrees . {envtmpdir}/linkcheck + + +[doc8] +ignore = + D002, + D004, +max-line-length = 88 + +[flake8] +max-line-length = 88 +ignore = + D ; ignores docstring style errors, enable if you are nit-picky + E203 ; ignores whitespace around : which is enforced by Black + W503 ; ignores linebreak before binary operator which is enforced by Black + RST304 ; ignores check for valid rst roles because it is too aggressive + T001 ; ignore print statements + RST301 ; ignores unexpected indentations in docstrings because it was not compatible with google style docstrings + RST203 ; gave false positives + RST202 ; gave false positives + RST201 ; gave false positives + W605 ; ignores regex relevant escape sequences + PT001 ; ignores brackets for fixtures. +per-file-ignores = + docs/source/conf.py:E501, E800 +warn-symbols = + pytest.mark.wip = Remove 'wip' mark for tests. + +[pytest] +addopts = --doctest-modules +markers = + wip: Tests that are work-in-progress. + slow: Tests that take a long time to run and are skipped in continuous integration. +norecursedirs = + docs + .tox