From 8bf0e2abcb20e6d7577190983ceedd56754e1ccb Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 30 Mar 2021 13:44:22 -0700 Subject: [PATCH] v1.6 (#56) --- .coveragerc | 4 - .github/workflows/code-style.yml | 21 ++ .github/workflows/coverage.yml | 36 ++++ .github/workflows/cross-compatibility.yml | 31 +++ .github/workflows/installation.yml | 49 +++++ .github/workflows/unit-tests.yml | 28 +++ .travis.yml | 23 -- HISTORY.rst | 6 + LICENSE.txt | 2 +- README.rst | 3 +- bin/build_docs.sh | 58 ------ docs/source/Makefile | 177 ---------------- docs/source/conf.py | 6 +- docs/source/core.rst | 51 ++++- docs/source/make.bat | 242 ---------------------- docs/source/server.rst | 2 + orca/__init__.py | 6 +- orca/orca.py | 172 +++++++++++++++ orca/tests/test_orca.py | 102 +++++++++ setup.cfg | 5 - setup.py | 5 +- 21 files changed, 498 insertions(+), 531 deletions(-) delete mode 100644 .coveragerc create mode 100644 .github/workflows/code-style.yml create mode 100644 .github/workflows/coverage.yml create mode 100644 .github/workflows/cross-compatibility.yml create mode 100644 .github/workflows/installation.yml create mode 100644 .github/workflows/unit-tests.yml delete mode 100644 .travis.yml delete mode 100755 bin/build_docs.sh delete mode 100644 docs/source/Makefile delete mode 100644 docs/source/make.bat delete mode 100644 setup.cfg diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index d3d73d7..0000000 --- a/.coveragerc +++ /dev/null @@ -1,4 +0,0 @@ -[run] -omit = - urbansim/urbanchoice/pmat.py - urbansim/**/tests/* diff --git a/.github/workflows/code-style.yml b/.github/workflows/code-style.yml new file mode 100644 index 0000000..bc852cd --- /dev/null +++ b/.github/workflows/code-style.yml @@ -0,0 +1,21 @@ +name: Code style + +# This workflow runs code style checks. + +on: + push: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Check code style + run: | + pip install pycodestyle + pycodestyle orca --max-line-length 100 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000..4cdafc5 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,36 @@ +name: Coverage + +# This workflow generates a coverage report (how much of the codebase is covered by the +# unit tests) and posts headline metrics to the PR thread. + +on: + # push: + pull_request: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install Orca + run: | + pip install . + pip install flask pygments # for server component tests + - name: Generate coverage report + run: | + pip install pytest coverage + coverage run --source orca --module pytest --verbose + coverage report --show-missing + echo "coverage=$(coverage report | grep '^TOTAL' | grep -oE '[^ ]+$')" >> $GITHUB_ENV + - name: Post comment on PR + uses: unsplash/comment-on-pr@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + msg: "Test coverage is ${{ env.coverage }}" + check_for_duplicate_msg: true diff --git a/.github/workflows/cross-compatibility.yml b/.github/workflows/cross-compatibility.yml new file mode 100644 index 0000000..2858e48 --- /dev/null +++ b/.github/workflows/cross-compatibility.yml @@ -0,0 +1,31 @@ +name: Cross-compatibility + +# This workflow runs the unit tests across a range of Python versions and operating +# systems. + +on: + # push: + pull_request: + workflow_dispatch: + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: [3.6, 3.7, 3.8] # add 3.9 when PyTables is available + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install Orca + run: | + pip install . + pip install flask pygments # for server component tests + - name: Run unit tests + run: | + pip install pytest + pytest -s diff --git a/.github/workflows/installation.yml b/.github/workflows/installation.yml new file mode 100644 index 0000000..1ea9f5d --- /dev/null +++ b/.github/workflows/installation.yml @@ -0,0 +1,49 @@ +name: Installation + +# This workflow tests installation from Pip and Conda across a range of Python versions +# and operating systems. You can run this manually after a new release is posted to +# confirm that it installs smoothly. This workflow also runs periodically in the +# background to catch dependency updates that cause problems. + +on: + # push: + # pull_request: + workflow_dispatch: + schedule: + - cron: '0 3 * * 1' # every Monday at 3am UTC (Sunday evening Calif time) + +jobs: + pip: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + python-version: [3.6, 3.7, 3.8] # add 3.9 when PyTables is available + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install Orca + run: | + pip install orca + + conda: + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash -l {0} # needed for conda persistence + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + python-version: [3.6, 3.7, 3.8, 3.9] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: conda-incubator/setup-miniconda@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install Orca + run: | + conda install orca --channel conda-forge diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 0000000..751c4ab --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,28 @@ +name: Unit tests + +# This workflow runs the unit tests in a single generic environment (recent but stable +# Python version on recent but stable Ubuntu). The cross-compatibility.yml workflow runs +# the same tests across multiple platforms. + +on: + push: + # pull_request: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install Orca + run: | + pip install . + pip install flask pygments # for server component tests + - name: Run unit tests + run: | + pip install pytest + pytest -s diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 52005b6..0000000 --- a/.travis.yml +++ /dev/null @@ -1,23 +0,0 @@ -language: python - -python: - - '2.7' - - '3.5' - - '3.6' - - '3.7' - - '3.8' - -install: - - pip install . - - pip install pytest-cov coveralls pycodestyle - - pip install flask pygments # for server component tests - - bin/build_js_bundle.sh complete - - pip list - - pip show orca - -script: - - pycodestyle orca - - py.test --cov orca --cov-report term-missing - -after_success: - - coveralls diff --git a/HISTORY.rst b/HISTORY.rst index 00329e9..a4b24c5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,3 +1,9 @@ +v1.6 +==== + +* New functionality to clear cache and update cache scope: https://udst.github.io/orca/core.html#caching +* Server module deprecated + v1.5.4 ====== diff --git a/LICENSE.txt b/LICENSE.txt index 8842884..266dce7 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2019, UrbanSim Inc. All rights reserved. +Copyright (c) 2021, UrbanSim Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/README.rst b/README.rst index 2dc3dbd..6790cf1 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,4 @@ -.. image:: https://coveralls.io/repos/UDST/orca/badge.svg?branch=master - :target: https://coveralls.io/r/UDST/orca?branch=master +.. image:: https://img.shields.io/badge/coverage-97%25-green :alt: Coverage Orca diff --git a/bin/build_docs.sh b/bin/build_docs.sh deleted file mode 100755 index b0e9d08..0000000 --- a/bin/build_docs.sh +++ /dev/null @@ -1,58 +0,0 @@ -#! /usr/bin/env bash - -# Copied from github.com/sympy/sympy -# -# This file automatically deploys changes to http://udst.github.io/orca/. -# This will only happen when building a non-pull request build on the master -# branch of Orca. -# It requires an access token which should be present in .travis.yml file. -# -# Following is the procedure to get the access token: -# -# $ curl -X POST -u -H "Content-Type: application/json" -d\ -# "{\"scopes\":[\"public_repo\"],\"note\":\"token for pushing from travis\"}"\ -# https://api.github.com/authorizations -# -# It'll give you a JSON response having a key called "token". -# -# $ gem install travis -# $ travis encrypt -r sympy/sympy GH_TOKEN= env.global -# -# This will give you an access token("secure"). This helps in creating an -# environment variable named GH_TOKEN while building. -# -# Add this secure code to .travis.yml as described here http://docs.travis-ci.com/user/encryption-keys/ - -# Exit on error -set -e - -ACTUAL_TRAVIS_JOB_NUMBER=`echo $TRAVIS_JOB_NUMBER| cut -d'.' -f 2` - -if [ "$TRAVIS_REPO_SLUG" == "UDST/orca" ] && \ - [ "$TRAVIS_BRANCH" == "master" ] && \ - [ "$TRAVIS_PULL_REQUEST" == "false" ] && \ - [ "$ACTUAL_TRAVIS_JOB_NUMBER" == "1" ]; then - - echo "Building docs" - cd docs - make clean - make html - - cd ../../ - echo "Setting git attributes" - git config --global user.email "jiffyclub@gmail.com" - git config --global user.name "Matt Davis" - - echo "Cloning repository" - git clone --quiet --single-branch --branch=gh-pages https://${GH_TOKEN}@github.com/UDST/orca.git gh-pages > /dev/null 2>&1 - - cd gh-pages - rm -rf * - cp -R ../orca/docs/_build/html/* ./ - touch .nojekyll - git add -A . - - git commit -am "Update dev doc after building $TRAVIS_BUILD_NUMBER" - echo "Pushing commit" - git push -fq origin gh-pages > /dev/null 2>&1 -fi diff --git a/docs/source/Makefile b/docs/source/Makefile deleted file mode 100644 index 27a06dc..0000000 --- a/docs/source/Makefile +++ /dev/null @@ -1,177 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build - -# User-friendly check for sphinx-build -ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) -$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) -endif - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " xml to make Docutils-native XML files" - @echo " pseudoxml to make pseudoxml-XML files for display purposes" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Orca.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Orca.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/Orca" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Orca" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -latexpdfja: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through platex and dvipdfmx..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." - -xml: - $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml - @echo - @echo "Build finished. The XML files are in $(BUILDDIR)/xml." - -pseudoxml: - $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml - @echo - @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/docs/source/conf.py b/docs/source/conf.py index 842d7e3..62820b1 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -51,16 +51,16 @@ # General information about the project. project = 'Orca' -copyright = '2020, UrbanSim Inc' +copyright = '2021, UrbanSim Inc' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '1.5.4' +version = '1.6' # The full version, including alpha/beta/rc tags. -release = '1.5.4' +release = '1.6' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/core.rst b/docs/source/core.rst index c92a50b..1043488 100644 --- a/docs/source/core.rst +++ b/docs/source/core.rst @@ -316,12 +316,17 @@ The three scope settings are: * ``'step'`` - Results are cached until the current pipeline step finishes. -Managing the Cache -~~~~~~~~~~~~~~~~~~ +An item's cache scope can be modified using +:py:func:`~orca.orca.update_injectable_scope`, +:py:func:`~orca.orca.update_table_scope`, or +:py:func:`~orca.orca.update_column_scope`. Omitting the scope or passing ``None`` +turns caching off for the item. These functions were added in Orca v1.6. -We hope that users will be able to do most of their cache management via -cache scopes, but there may be situations, especially during testing, -when more manual management is required. +Disabling Caching +~~~~~~~~~~~~~~~~~ + +There may be situations, especially during testing, +that require disabling the caching system. Caching can be turned off globally using the :py:func:`~orca.orca.disable_cache` function @@ -334,8 +339,32 @@ context manager:: with orca.cache_disabled(): result = orca.eval_variable('my_table') -Finally, users can manually clear the cache using -:py:func:`~orca.orca.clear_cache`. +Manually Clearing Cache +~~~~~~~~~~~~~~~~~~~~~~~ + +Orca's entire cache can be cleared using :py:func:`~orca.orca.clear_cache`. + +Cache can also be cleared manually for individual items, to allow finer control +over re-computation. These functions were added in Orca v1.6. + +To clear the cached value of an injectable, use +:py:func:`~orca.orca.clear_injectable`. To clear the cached copy of an entire +table, use :py:func:`~orca.orca.clear_table`. + +A dynamically generated column can be cleared using +:py:func:`~orca.orca.clear_column`:: + + orca.clear_column('my_table', 'my_col') + +To clear all dynamically generated columns from a table, use +:py:func:`~orca.orca.clear_columns`:: + + orca.clear_columns('my_table') + +Or clear a subset of the columns like this:: + + orca.clear_columns('my_table', ['col1', 'col2']) + Steps ----- @@ -669,7 +698,13 @@ Cache API disable_cache enable_cache cache_on - cache_disabled + clear_injectable + clear_table + clear_column + clear_columns + update_injectable_scope + update_table_scope + update_column_scope API Docs ~~~~~~~~ diff --git a/docs/source/make.bat b/docs/source/make.bat deleted file mode 100644 index 02b37ca..0000000 --- a/docs/source/make.bat +++ /dev/null @@ -1,242 +0,0 @@ -@ECHO OFF - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set BUILDDIR=_build -set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . -set I18NSPHINXOPTS=%SPHINXOPTS% . -if NOT "%PAPER%" == "" ( - set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% - set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% -) - -if "%1" == "" goto help - -if "%1" == "help" ( - :help - echo.Please use `make ^` where ^ is one of - echo. html to make standalone HTML files - echo. dirhtml to make HTML files named index.html in directories - echo. singlehtml to make a single large HTML file - echo. pickle to make pickle files - echo. json to make JSON files - echo. htmlhelp to make HTML files and a HTML help project - echo. qthelp to make HTML files and a qthelp project - echo. devhelp to make HTML files and a Devhelp project - echo. epub to make an epub - echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter - echo. text to make text files - echo. man to make manual pages - echo. texinfo to make Texinfo files - echo. gettext to make PO message catalogs - echo. changes to make an overview over all changed/added/deprecated items - echo. xml to make Docutils-native XML files - echo. pseudoxml to make pseudoxml-XML files for display purposes - echo. linkcheck to check all external links for integrity - echo. doctest to run all doctests embedded in the documentation if enabled - goto end -) - -if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* - goto end -) - - -%SPHINXBUILD% 2> nul -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/html. - goto end -) - -if "%1" == "dirhtml" ( - %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. - goto end -) - -if "%1" == "singlehtml" ( - %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. - goto end -) - -if "%1" == "pickle" ( - %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the pickle files. - goto end -) - -if "%1" == "json" ( - %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the JSON files. - goto end -) - -if "%1" == "htmlhelp" ( - %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run HTML Help Workshop with the ^ -.hhp project file in %BUILDDIR%/htmlhelp. - goto end -) - -if "%1" == "qthelp" ( - %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run "qcollectiongenerator" with the ^ -.qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\Orca.qhcp - echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\Orca.ghc - goto end -) - -if "%1" == "devhelp" ( - %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. - goto end -) - -if "%1" == "epub" ( - %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub file is in %BUILDDIR%/epub. - goto end -) - -if "%1" == "latex" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdf" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdfja" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf-ja - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "text" ( - %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The text files are in %BUILDDIR%/text. - goto end -) - -if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. - goto end -) - -if "%1" == "texinfo" ( - %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. - goto end -) - -if "%1" == "gettext" ( - %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The message catalogs are in %BUILDDIR%/locale. - goto end -) - -if "%1" == "changes" ( - %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes - if errorlevel 1 exit /b 1 - echo. - echo.The overview file is in %BUILDDIR%/changes. - goto end -) - -if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck - if errorlevel 1 exit /b 1 - echo. - echo.Link check complete; look for any errors in the above output ^ -or in %BUILDDIR%/linkcheck/output.txt. - goto end -) - -if "%1" == "doctest" ( - %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest - if errorlevel 1 exit /b 1 - echo. - echo.Testing of doctests in the sources finished, look at the ^ -results in %BUILDDIR%/doctest/output.txt. - goto end -) - -if "%1" == "xml" ( - %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The XML files are in %BUILDDIR%/xml. - goto end -) - -if "%1" == "pseudoxml" ( - %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. - goto end -) - -:end diff --git a/docs/source/server.rst b/docs/source/server.rst index ee6ecf9..8110abd 100644 --- a/docs/source/server.rst +++ b/docs/source/server.rst @@ -1,6 +1,8 @@ Orca Server =========== +*THE ORCA SERVER MODULE IS DEPRECATED AND WILL BE REMOVED IN A FUTURE RELEASE.* + Orca ships with a `Flask `__ server that can provide data about and from an Orca configuration. You can use it as a zero-configuration server for your data use the diff --git a/orca/__init__.py b/orca/__init__.py index fad5330..3058f4c 100644 --- a/orca/__init__.py +++ b/orca/__init__.py @@ -1,7 +1,3 @@ -# Orca -# Copyright (C) 2016 UrbanSim Inc. -# See full license in LICENSE. - from .orca import * -version = __version__ = '1.5.4' +version = __version__ = '1.6' diff --git a/orca/orca.py b/orca/orca.py index 5801f50..3955724 100644 --- a/orca/orca.py +++ b/orca/orca.py @@ -96,6 +96,178 @@ def clear_cache(scope=None): logger.debug('cleared cached values with scope {!r}'.format(scope)) +def clear_injectable(injectable_name): + """ + Clear the cached value of an injectable. *Added in Orca v1.6.* + + Parameters + ---------- + name: str + Name of injectable to clear. + + """ + _INJECTABLES[injectable_name].clear_cached() + + +def clear_table(table_name): + """ + Clear the cached copy of an entire table. *Added in Orca v1.6.* + + Parameters + ---------- + name: str + Name of table to clear. + + """ + _TABLES[table_name].clear_cached() + + +def clear_column(table_name, column_name): + """ + Clear the cached copy of a dynamically generated column. + *Added in Orca v1.6.* + + Parameters + ---------- + table_name: str + Table containing the column to clear. + column_name: str + Name of the column to clear. + + """ + _COLUMNS[(table_name, column_name)].clear_cached() + + +def clear_columns(table_name, columns=None): + """ + Clear all (or a specified list) of the dynamically generated columns + associated with a table. *Added in Orca v1.6.* + + Parameters + ---------- + table_name: str + Table name. + columns: list of str, optional, default None + List of columns to clear. If None, all extra/computed + columns in the table will be cleeared. + + """ + if columns is None: + tab = get_table(table_name) + cols = tab.columns + local_cols = tab.local_columns + columns = [c for c in cols if c not in local_cols] + print('****************') + print(columns) + + for col in columns: + clear_column(table_name, col) + + +def _update_scope(wrapper, new_scope=None): + """ + Update the cache scope for a wrapper (in place). + *Added in Orca v1.6.* + + Parameters + ---------- + wrapper: object + Should be an instance of wrapper with attributes + `cache`, `cache_scope` and method `clear_cached`. + new_scope: str, optional default None + The new scope value. None implies no caching. + + """ + # allowable scopes, values indicate the update granularity + scopes = { + None: 0, + _CS_STEP: 1, + _CS_ITER: 2, + _CS_FOREVER: 3 + } + if new_scope not in scopes.keys(): + msg = '{} is not an allowed cache scope, '.format(new_scope) + msg += 'allowed scopes are {}'.format(list(scopes.keys())) + raise ValueError(msg) + + # update the cache properties + curr_cache = wrapper.cache + curr_scope = wrapper.cache_scope + if new_scope is None: + # set to defaults, i.e. no caching + wrapper.cache = False + wrapper.cache_scope = _CS_FOREVER + else: + wrapper.cache = True + wrapper.cache_scope = new_scope + + # clear out any existing caches if the provided scope is + # more granular than the existing + old_granularity = scopes[curr_scope] + new_granularity = scopes[new_scope] + if new_granularity < old_granularity: + wrapper.clear_cached() + + +def update_injectable_scope(name, new_scope=None): + """ + Update the cache scope for a wrapped injectable function. + Clears the cache if the new scope is more granular + than the existing. *Added in Orca v1.6.* + + Parameters + ---------- + name: str + Name of the injectable to update. + new_scope: str, optional default None + Valid values: None, 'forever', 'iteration', 'step' + None implies no caching. + + """ + _update_scope( + get_raw_injectable(name), new_scope) + + +def update_column_scope(table_name, column_name, new_scope=None): + """ + Update the cache scope for a wrapped column function. Clears + the cache if the new scope is more granular than the existing. + *Added in Orca v1.6.* + + Parameters + ---------- + table_name: str + Name of the table. + column_name: str + Name of the column to update. + new_scope: str, optional default None + Valid values: None, 'forever', 'iteration', 'step' + None implies no caching. + + """ + _update_scope( + get_raw_column(table_name, column_name), new_scope) + + +def update_table_scope(name, new_scope=None): + """ + Update the cache scope for a wrapped table function. Clears + the cache if the new scope is more granular than the existing. + *Added in Orca v1.6.* + + Parameters + ---------- + name: str + Name of the table to update. + new_scope: str, optional default None + Valid values: None, 'forever', 'iteration', 'step' + None implies no caching. + + """ + _update_scope( + get_raw_table(name), new_scope) + + def enable_cache(): """ Allow caching of registered variables that explicitly have diff --git a/orca/tests/test_orca.py b/orca/tests/test_orca.py index dd1592f..e13ebae 100644 --- a/orca/tests/test_orca.py +++ b/orca/tests/test_orca.py @@ -323,6 +323,108 @@ def c(): pdt.assert_series_equal(c()(), series * 6) +def test_manual_cache_clearing(df): + + @orca.injectable(cache=True) + def my_inj(x): + return x * 2 + + @orca.table(cache=True) + def my_table(x): + return df + x + + @orca.column('my_table', cache=True) + def extra1(my_table): + return my_table['a'] * -1 + + @orca.column('my_table', cache=True) + def extra2(my_table): + return my_table['b'] * -1 + + def run_checks(x): + orca.add_injectable('x', x) + inj = orca.get_injectable('my_inj') + tab = orca.get_table('my_table').to_frame() + + assert inj == x * 2 + assert (tab['a'] == df['a'] + x).all() + assert (tab['b'] == df['b'] + x).all() + assert (tab['extra1'] == -1 * (df['a'] + x)).all() + assert (tab['extra2'] == -1 * (df['b'] + x)).all() + + # initial collection + run_checks(100) + + # manually clear out and re-check + orca.clear_injectable('my_inj') + orca.clear_column('my_table', 'extra1') + orca.clear_column('my_table', 'extra2') + orca.clear_table('my_table') + run_checks(200) + + # check clearing all columns + orca.clear_injectable('my_inj') + orca.clear_columns('my_table') + orca.clear_table('my_table') + run_checks(300) + + # check clearing subset of columns + orca.clear_injectable('my_inj') + orca.clear_columns('my_table', ['extra1', 'extra2']) + orca.clear_table('my_table') + run_checks(400) + + +def test_update_scope(): + + @orca.injectable(cache=True) + def my_inj(x): + return x + + @orca.table() + def my_table(x): + df = pd.DataFrame({'a': [100, 200, 300]}) + return df + x + + @orca.column('my_table', cache=True) + def my_col(my_table): + return my_table['a'] * -1 + + # initial collection + orca.add_injectable('x', 10) + orca.get_injectable('my_inj') + orca.get_table('my_table').to_frame() + + # update injectable scope + orca.update_injectable_scope('my_inj', 'iteration') + inj = orca.get_raw_injectable('my_inj') + assert inj.cache + assert inj.cache_scope == 'iteration' + + # update table scope + orca.update_table_scope('my_table', 'step') + tab = orca.get_raw_table('my_table') + assert tab.cache + assert tab.cache_scope == 'step' + + # update column scope + orca.update_column_scope('my_table', 'my_col') + col = orca.get_raw_column('my_table', 'my_col') + assert not col.cache + assert col.cache_scope == 'forever' + + # invalid cache scope + with pytest.raises(ValueError): + orca.update_table_scope('my_table', 'bogus scope') + + # make sure the cached values got cleared + orca.add_injectable('x', 20) + assert orca.get_injectable('my_inj') == 20 + df = orca.get_table('my_table').to_frame() + assert (df['a'].values == [120, 220, 320]).all() + assert (df['my_col'].values == [-120, -220, -320]).all() + + def test_column_cache_disabled(df): orca.add_injectable('x', 2) series = pd.Series([1, 2, 3], index=['x', 'y', 'z']) diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index e13865d..0000000 --- a/setup.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[pycodestyle] -max-line-length = 100 - -[bdist_wheel] -universal=1 diff --git a/setup.py b/setup.py index e0dd683..4e093b9 100644 --- a/setup.py +++ b/setup.py @@ -38,15 +38,14 @@ def run(self): setup( name='orca', - version='1.5.4', - description='A pipeline orchestration tool with Pandas support', + version='1.6', + description='Python library for task orchestration', long_description=long_description, author='UrbanSim Inc.', author_email='info@urbansim.com', license='BSD', url='https://github.com/udst/orca', classifiers=[ - 'Development Status :: 4 - Beta', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6',