From 5fc9c037a926a561fd9d26cd8347b5c69a5b0def Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 17 Aug 2020 14:13:42 -0700 Subject: [PATCH 1/9] path -> path_or_buf --- orca/server/tests/test_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orca/server/tests/test_server.py b/orca/server/tests/test_server.py index cca56a4..16c6046 100644 --- a/orca/server/tests/test_server.py +++ b/orca/server/tests/test_server.py @@ -233,7 +233,7 @@ def test_column_csv(tapp, dfa): assert rv.status_code == 200 data = rv.data.decode('utf-8') - assert data == dfa.a.to_csv(path=None) + assert data == dfa.a.to_csv(path_or_buf=None) def test_no_column_404(tapp): From 2f67c8d33ef3a825192949b863d2f812eccf1f83 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 17 Aug 2020 14:19:19 -0700 Subject: [PATCH 2/9] path -> path_or_buf --- orca/server/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orca/server/server.py b/orca/server/server.py index fd51caa..1a55402 100644 --- a/orca/server/server.py +++ b/orca/server/server.py @@ -336,7 +336,7 @@ def column_csv(table_name, col_name): Return a column as CSV using Pandas' default CSV output. """ - csv = orca.get_table(table_name).get_column(col_name).to_csv(path=None) + csv = orca.get_table(table_name).get_column(col_name).to_csv(path_or_buf=None) return csv, 200, {'Content-Type': 'text/csv'} From d90125b8a211ac62a111d18d2e56ee660a6203d9 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 17 Aug 2020 16:37:20 -0700 Subject: [PATCH 3/9] Travis cleanup --- .travis.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index b0eb60e..52005b6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,14 +5,13 @@ python: - '3.5' - '3.6' - '3.7' - # - '3.8' broken because PyTables binaries aren't available yet + - '3.8' install: - pip install . - pip install pytest-cov coveralls pycodestyle - pip install flask pygments # for server component tests - bin/build_js_bundle.sh complete - - pip install numpydoc sphinx sphinx_rtd_theme # for building docs - pip list - pip show orca @@ -22,8 +21,3 @@ script: after_success: - coveralls - - bin/build_docs.sh - -env: - global: - secure: DMQgAnged8quDKf3pTLG6RZVMKhrhZfYSRKaPFJrERXEOuRPJjC4QYG24pdyCV2aOQbSOiF16vSctpDmMfjLDARkhNm8FvYYCOKLkE+4ijQ4vE/uQaHsnosGS/1LOOFGjK0h2dvm4CNJqV1JUhCKeOVe/87GGY2JVBy+NJnsl70= From 5441813b16b7bdb4e1631c2c887057c0a3caea0f Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 30 Sep 2020 10:52:54 -0700 Subject: [PATCH 4/9] Resolving deprecations --- orca/orca.py | 13 +++++++++---- orca/server/server.py | 2 +- orca/server/tests/test_server.py | 7 ++++--- orca/tests/test_orca.py | 2 +- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/orca/orca.py b/orca/orca.py index c4a4f53..2d0793f 100644 --- a/orca/orca.py +++ b/orca/orca.py @@ -11,17 +11,22 @@ import logging import time import warnings -from collections import Callable, namedtuple +from collections import namedtuple +from collections.abc import Callable from contextlib import contextmanager from functools import wraps +try: + from tlz.compatibility import zip as zip # for Python 2.7 +except ImportError: + pass + import pandas as pd import tables import tlz as tz from . import utils from .utils.logutil import log_start_finish -from collections import namedtuple warnings.filterwarnings('ignore', category=tables.NaturalNameWarning) logger = logging.getLogger(__name__) @@ -937,8 +942,8 @@ def _collect_variables(names, expressions=None): expressions = [] offset = len(names) - len(expressions) labels_map = dict(tz.concatv( - tz.compatibility.zip(names[:offset], names[:offset]), - tz.compatibility.zip(names[offset:], expressions))) + zip(names[:offset], names[:offset]), + zip(names[offset:], expressions))) all_variables = tz.merge(_INJECTABLES, _TABLES) variables = {} diff --git a/orca/server/server.py b/orca/server/server.py index fd51caa..1a55402 100644 --- a/orca/server/server.py +++ b/orca/server/server.py @@ -336,7 +336,7 @@ def column_csv(table_name, col_name): Return a column as CSV using Pandas' default CSV output. """ - csv = orca.get_table(table_name).get_column(col_name).to_csv(path=None) + csv = orca.get_table(table_name).get_column(col_name).to_csv(path_or_buf=None) return csv, 200, {'Content-Type': 'text/csv'} diff --git a/orca/server/tests/test_server.py b/orca/server/tests/test_server.py index cca56a4..b0cb22c 100644 --- a/orca/server/tests/test_server.py +++ b/orca/server/tests/test_server.py @@ -1,8 +1,9 @@ import json import orca +import numpy as np import pandas as pd -import pandas.util.testing as pdt +import pandas.testing as pdt import pytest from .. import server @@ -233,7 +234,7 @@ def test_column_csv(tapp, dfa): assert rv.status_code == 200 data = rv.data.decode('utf-8') - assert data == dfa.a.to_csv(path=None) + assert data == dfa.a.to_csv(path_or_buf=None) def test_no_column_404(tapp): @@ -420,5 +421,5 @@ def test_table_groupbyagg_level_std(tapp): pdt.assert_series_equal( test, pd.Series( - [pd.np.nan, pd.Series([80, 90]).std()], + [np.nan, pd.Series([80, 90]).std()], index=['a', 'b'], name='b')) diff --git a/orca/tests/test_orca.py b/orca/tests/test_orca.py index aae3c30..dd1592f 100644 --- a/orca/tests/test_orca.py +++ b/orca/tests/test_orca.py @@ -6,8 +6,8 @@ import tempfile import pandas as pd +import pandas.testing as pdt import pytest -from pandas.util import testing as pdt from .. import orca from ..utils.testing import assert_frames_equal From 62fbdd246353d9a204f7479f270a2231c76d0060 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Wed, 30 Sep 2020 11:19:02 -0700 Subject: [PATCH 5/9] Switch to core 'zip' --- orca/orca.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/orca/orca.py b/orca/orca.py index 2d0793f..c864c00 100644 --- a/orca/orca.py +++ b/orca/orca.py @@ -16,11 +16,6 @@ from contextlib import contextmanager from functools import wraps -try: - from tlz.compatibility import zip as zip # for Python 2.7 -except ImportError: - pass - import pandas as pd import tables import tlz as tz From e013787a493eed87a7788b5b645b1b562769f329 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 2 Nov 2020 21:06:09 -0800 Subject: [PATCH 6/9] Cross-python compatibility --- orca/orca.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/orca/orca.py b/orca/orca.py index c864c00..5801f50 100644 --- a/orca/orca.py +++ b/orca/orca.py @@ -12,10 +12,14 @@ import time import warnings from collections import namedtuple -from collections.abc import Callable +try: + from collections.abc import Callable +except ImportError: # Python 2.7 + from collections import Callable from contextlib import contextmanager from functools import wraps + import pandas as pd import tables import tlz as tz From 8d5352f36e653a432b020cce085372e9ec8b36f6 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Mon, 2 Nov 2020 21:15:14 -0800 Subject: [PATCH 7/9] Versioning --- orca/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/orca/__init__.py b/orca/__init__.py index fcd9ce5..4326311 100644 --- a/orca/__init__.py +++ b/orca/__init__.py @@ -4,4 +4,4 @@ from .orca import * -version = __version__ = '1.5.3' +version = __version__ = '1.5.4.dev0' diff --git a/setup.py b/setup.py index 211fc82..8abe2b0 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ def run(self): setup( name='orca', - version='1.5.3', + version='1.5.4.dev0', description='A pipeline orchestration tool with Pandas support', long_description=long_description, author='UrbanSim Inc.', From b1454b9b679d9d1bc15932dd658c6c2db5383704 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 5 Nov 2020 10:09:49 -0800 Subject: [PATCH 8/9] Release prep --- HISTORY.rst | 5 +++++ README.rst | 39 ++++++++++------------------------- docs/README.md | 31 ++++++++++++++++++++++++++++ docs/build/.gitignore | 1 + docs/{ => source}/Makefile | 0 docs/{ => source}/conf.py | 6 +++--- docs/{ => source}/core.rst | 0 docs/{ => source}/example.rst | 0 docs/{ => source}/index.rst | 13 +++++------- docs/{ => source}/make.bat | 0 docs/{ => source}/server.rst | 13 ++++++++++++ docs/{ => source}/utils.rst | 0 orca/__init__.py | 2 +- setup.py | 3 ++- 14 files changed, 72 insertions(+), 41 deletions(-) create mode 100644 docs/README.md create mode 100644 docs/build/.gitignore rename docs/{ => source}/Makefile (100%) rename docs/{ => source}/conf.py (99%) rename docs/{ => source}/core.rst (100%) rename docs/{ => source}/example.rst (100%) rename docs/{ => source}/index.rst (90%) rename docs/{ => source}/make.bat (100%) rename docs/{ => source}/server.rst (95%) rename docs/{ => source}/utils.rst (100%) diff --git a/HISTORY.rst b/HISTORY.rst index 7bbf149..00329e9 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,3 +1,8 @@ +v1.5.4 +====== + +* Support for PyToolz 0.11+ + v1.5.3 ====== diff --git a/README.rst b/README.rst index 9959222..110128f 100644 --- a/README.rst +++ b/README.rst @@ -1,40 +1,23 @@ Orca ==== -.. image:: https://img.shields.io/pypi/v/orca.svg - :target: https://pypi.python.org/pypi/orca/ - :alt: PyPI Latest Version - -.. image:: https://travis-ci.org/UDST/orca.svg?branch=master - :target: https://travis-ci.org/UDST/orca - :alt: Build Status - .. image:: https://coveralls.io/repos/UDST/orca/badge.svg?branch=master :target: https://coveralls.io/r/UDST/orca?branch=master :alt: Coverage -.. image:: https://img.shields.io/pypi/wheel/orca.svg - :target: https://pypi.python.org/pypi/orca/ - :alt: Wheel Status +Orca is a Python library for task orchestration. It's designed for workflows like city simulation, where the data representing a model's state is so large that it needs to be managed outside of the task graph. + +The building blocks of a workflow are "steps", Python functions that can be assembled on the fly into linear or cyclical pipelines. Steps typically interact with a central data store that persists in memory while the pipeline runs. Derived tables and columns can be updated automatically as base data changes, and pipeline components are evaluated lazily to reduce unnecessary overhead. -Orca is a pipeline orchestration tool that allows you to define dynamic data -sources and explicitly connect them to processing functions. -Orca has many features for working with `Pandas `__ -data structures, but it can be used with anything. +Orca is used in `UrbanSim ` and other projects. -Learn more in the official docs at https://udst.github.io/orca/. +Documentation +------------- -Building the Orca UI JS Bundle ------------------------------- +- `udst.github.io/orca/ ` -Orca ships with a bundle of JavaScript for the server UI. -If you've installed Orca from ``pip`` or ``conda`` you already have the -bundle, but if you're working on Orca you might need to build it manually: +Installation +------------ -* Make sure `nodejs `__ is installed. - (I use `Homebrew `__ on my Mac.) -* Install `gulp `__: ``npm install -g gulp`` -* Change directories to ``orca/server/static`` -* Run ``npm install`` to install dependencies -* Build the bundle: ``gulp js-build``, or -* Watch JS files to rebuild the bundle on changes: ``gulp js-watch`` +- ``pip install orca`` or +- ``conda install orca --channel conda-forge`` diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..f030d8c --- /dev/null +++ b/docs/README.md @@ -0,0 +1,31 @@ +This folder generates the Orca online documentation, hosted at https://udst.github.io/orca/. + +### How it works + +HTML files are generated using [Sphinx](http://sphinx-doc.org) and hosted with GitHub Pages from the `gh-pages` branch of the repository. The online documentation is rendered and updated **manually**. + +### Editing the documentation + +The files in `docs/source`, along with docstrings in the source code, determine what appears in the rendered documentation. Here's a [good tutorial](https://pythonhosted.org/an_example_pypi_project/sphinx.html) for Sphinx. + +### Previewing changes locally + +Install the copy of Orca that the documentation is meant to reflect. Install the documentation tools. + +``` +pip install . +pip install sphinx sphinx_rtd_theme numpydoc +``` + +Build the documentation. There should be status messages and warnings, but no errors. + +``` +cd docs +sphinx-build -b html source build +``` + +The HTML files will show up in `docs/build/`. + +### Uploading changes + +Clone a second copy of the repository and check out the `gh-pages` branch. Copy over the updated HTML files, commit them, and push the changes to GitHub. diff --git a/docs/build/.gitignore b/docs/build/.gitignore new file mode 100644 index 0000000..150f68c --- /dev/null +++ b/docs/build/.gitignore @@ -0,0 +1 @@ +*/* diff --git a/docs/Makefile b/docs/source/Makefile similarity index 100% rename from docs/Makefile rename to docs/source/Makefile diff --git a/docs/conf.py b/docs/source/conf.py similarity index 99% rename from docs/conf.py rename to docs/source/conf.py index 88af7e1..842d7e3 100644 --- a/docs/conf.py +++ b/docs/source/conf.py @@ -51,16 +51,16 @@ # General information about the project. project = 'Orca' -copyright = '2019, UrbanSim Inc' +copyright = '2020, UrbanSim Inc' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '1.5' +version = '1.5.4' # The full version, including alpha/beta/rc tags. -release = '1.5.3' +release = '1.5.4' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/core.rst b/docs/source/core.rst similarity index 100% rename from docs/core.rst rename to docs/source/core.rst diff --git a/docs/example.rst b/docs/source/example.rst similarity index 100% rename from docs/example.rst rename to docs/source/example.rst diff --git a/docs/index.rst b/docs/source/index.rst similarity index 90% rename from docs/index.rst rename to docs/source/index.rst index e2269d8..c1fca87 100644 --- a/docs/index.rst +++ b/docs/source/index.rst @@ -119,19 +119,16 @@ Orca offers some conveniences for streamlining the construction of pipelines: Installation ------------ -Orca depends on Pandas, PyTables, and toolz (or cytoolz). -Pandas and PyTables are included in `Anaconda `__. -Toolz may be installed with `pip `__, -and both toolz and cytoolz are can be installed by -`conda `__ (if you are using Anaconda or miniconda). +Orca requires Pandas, PyTables, and PyToolz, which will be installed automatically if they are not already present in your Python environment. -With the dependencies installed, install Orca with pip:: + +You can install Orca with pip:: pip install orca -Orca may also be installed with conda:: +Or with conda:: - conda install -c udst orca + conda install orca --channel conda-forge Add the server option to include the optional server dependencies:: diff --git a/docs/make.bat b/docs/source/make.bat similarity index 100% rename from docs/make.bat rename to docs/source/make.bat diff --git a/docs/server.rst b/docs/source/server.rst similarity index 95% rename from docs/server.rst rename to docs/source/server.rst index e8a595f..ee6ecf9 100644 --- a/docs/server.rst +++ b/docs/source/server.rst @@ -22,6 +22,19 @@ Orca by including the ``[server]`` option:: pip install orca[server] +Building the Orca UI JS Bundle +------------------------------ + +This is NOT NECESSARY if you've installed Orca from ``pip`` or ``conda``, but these instructions may help if you're working directly with the Orca source code. + +* Make sure `nodejs `__ is installed. + (`Homebrew `__ is a nice way to do this on Mac.) +* Install `gulp `__: ``npm install -g gulp`` +* Change directories to ``orca/server/static`` +* Run ``npm install`` to install dependencies +* Build the bundle: ``gulp js-build``, or +* Watch JS files to rebuild the bundle on changes: ``gulp js-watch`` + Start the Server ---------------- diff --git a/docs/utils.rst b/docs/source/utils.rst similarity index 100% rename from docs/utils.rst rename to docs/source/utils.rst diff --git a/orca/__init__.py b/orca/__init__.py index 4326311..fad5330 100644 --- a/orca/__init__.py +++ b/orca/__init__.py @@ -4,4 +4,4 @@ from .orca import * -version = __version__ = '1.5.4.dev0' +version = __version__ = '1.5.4' diff --git a/setup.py b/setup.py index 8abe2b0..e0dd683 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ def run(self): setup( name='orca', - version='1.5.4.dev0', + version='1.5.4', description='A pipeline orchestration tool with Pandas support', long_description=long_description, author='UrbanSim Inc.', @@ -52,6 +52,7 @@ def run(self): 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', 'License :: OSI Approved :: BSD License' ], packages=find_packages(exclude=['*.tests']), From 36d62c9522df75536503fdfc30a83a3891fa7e02 Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Thu, 5 Nov 2020 10:14:28 -0800 Subject: [PATCH 9/9] Cleanup --- README.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 110128f..2dc3dbd 100644 --- a/README.rst +++ b/README.rst @@ -1,23 +1,23 @@ -Orca -==== - .. image:: https://coveralls.io/repos/UDST/orca/badge.svg?branch=master :target: https://coveralls.io/r/UDST/orca?branch=master :alt: Coverage +Orca +==== + Orca is a Python library for task orchestration. It's designed for workflows like city simulation, where the data representing a model's state is so large that it needs to be managed outside of the task graph. The building blocks of a workflow are "steps", Python functions that can be assembled on the fly into linear or cyclical pipelines. Steps typically interact with a central data store that persists in memory while the pipeline runs. Derived tables and columns can be updated automatically as base data changes, and pipeline components are evaluated lazily to reduce unnecessary overhead. -Orca is used in `UrbanSim ` and other projects. +Orca is used in `UrbanSim `__ and other projects. Documentation ------------- -- `udst.github.io/orca/ ` +- `udst.github.io/orca/ `__ Installation ------------ -- ``pip install orca`` or +- ``pip install orca`` - ``conda install orca --channel conda-forge``