updated dependencies; adopting black

Author: mclevey

.black.toml

@@ -0,0 +1,18 @@
+[tool.black]
+line-length = 88
+target-version = ['py38']
+include = '\.pyi?$'
+exclude = '''
+/(
+    \.git
+    | \.hg
+    | \.mypy_cache
+    | \.tox
+    | \.venv
+    | _build
+    | buck-out
+    | build
+    | dist
+    | \.pdpp.*
+)/
+'''

.flake8

@@ -0,0 +1,4 @@
+[flake8]
+exclude = .venv, */dodo.py
+max-line-length = 90
+extend-ignore = E203, W503, E231

.gitignore

@@ -1,3 +1,5 @@
+.DS_Store
+
 /.vscode*
 !/.gitignore
 *__pycache__
@@ -7,4 +9,4 @@
 /build/
 /*.egg-info
 /*.egg
-/.idea
+/.idea

.isort.cfg

@@ -0,0 +1,4 @@
+[settings]
+profile = black
+line_length = 88
+skip = .pdpp*

.pre-commit-config.yaml

@@ -0,0 +1,31 @@
+repos:
+  - repo: [email protected]:pre-commit/pre-commit-hooks.git
+    rev: v4.4.0
+    hooks:
+      - id: check-added-large-files
+        args: ['--maxkb=102400']
+      - id: check-executables-have-shebangs
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: check-toml
+      - id: check-yaml
+      - id: detect-private-key
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+
+  - repo: [email protected]:psf/black.git
+    rev: 23.1.0
+    hooks:
+      - id: black
+        args: ['--config', '.black.toml']
+
+  - repo: [email protected]:pre-commit/mirrors-isort.git
+    rev: v5.10.1
+    hooks:
+      - id: isort
+
+  - repo: [email protected]:pycqa/flake8.git
+    rev: 6.0.0
+    hooks:
+      - id: flake8
+        args: ['--config', '.flake8']

LICENSE.txt

@@ -20,7 +20,7 @@
 
 # `pdpp`
 
-`pdpp` is a command-line interface for facilitating the creation and maintainance of transparent and reproducible data workflows. `pdpp` adheres to principles espoused by Patrick Ball in his manifesto on ['Principled Data Processing'](https://www.youtube.com/watch?v=ZSunU9GQdcI). `pdpp` can be used to create 'tasks', populate task directories with the requisite subdirectories, link together tasks' inputs and outputs, and executing the pipeline using the `doit` [suite of automation tools](https://pydoit.org/). 
+`pdpp` is a command-line interface for facilitating the creation and maintainance of transparent and reproducible data workflows. `pdpp` adheres to principles espoused by Patrick Ball in his manifesto on ['Principled Data Processing'](https://www.youtube.com/watch?v=ZSunU9GQdcI). `pdpp` can be used to create 'tasks', populate task directories with the requisite subdirectories, link together tasks' inputs and outputs, and executing the pipeline using the `doit` [suite of automation tools](https://pydoit.org/).
 
 `pdpp` is also capable of producing rich visualizaitons of the data processing workflows it creates:
 
@@ -34,12 +34,12 @@ Each task directory contains at minimum three subdirectories:
 2. `output`, which contains all of the task's local data outputs (also referred to as 'targets')
 3. `src`, which all of the task's source codeWhich, ideally, would be contained within a single script file.]
 
-The `pdpp` package adds two additional constraints to Patrick Ball's original formulation of PDP: 
+The `pdpp` package adds two additional constraints to Patrick Ball's original formulation of PDP:
 
 1. All local data files needed by the workflow but which are not generated by any of the workflow's tasks must be included in the `_import_` directory, which `pdpp` places at the same directory level as the overall workflow during project initialization.
 2. All local data files produced by the workflow as project outputs must be routed into the `_export_` directory, which `pdpp` places at the same directory level as the overall workflow during project initialization.
 
-These additional constraints disambiguate the input and output of the overall workflow, which permits `pdpp` workflows to be embedded within one another. 
+These additional constraints disambiguate the input and output of the overall workflow, which permits `pdpp` workflows to be embedded within one another.
 
 
 ## Installation Prerequisites
@@ -67,15 +67,15 @@ Doing so should produce a directory tree similar to this one:
 
 ![](img/init.png)
 
-For the purposes of this example, a `.csv` file containing some toy data has been added to the `_import_` directory. 
+For the purposes of this example, a `.csv` file containing some toy data has been added to the `_import_` directory.
 
 At this point, we're ready to add our first task to the project. To do this, we'll use the `new` command:
 
 ```bash
 pdpp new
 ```
 
-Upon executing the command, `pdpp` will request a name for the new task. We'll call it 'task_1'. After supplying the name, `pdpp` will display an interactive menu which allows users to specify which other tasks in the project contain files that 'task_1' will depend upon. 
+Upon executing the command, `pdpp` will request a name for the new task. We'll call it 'task_1'. After supplying the name, `pdpp` will display an interactive menu which allows users to specify which other tasks in the project contain files that 'task_1' will depend upon.
 
 ![](img/task_1_task_dep.png)
 
@@ -96,7 +96,7 @@ new_rows = []
 
 with open('../input/example_data.csv', 'r') as f1:
     r = csv.reader(f1)
-    for row in r: 
+    for row in r:
         new_row = [int(row[0]) + 1, int(row[1]) + 1]
         new_rows.append(new_row)
 
@@ -112,7 +112,7 @@ After running `task_1.py`, a new file called `example_data_plus_one.csv` should
 pdpp rig
 ```
 
-Select `_export_` from the list of tasks available, then select `task_1` (and not `_import_`); finally, select `example_data_plus_one.csv` as the only dependency for `_export_`. 
+Select `_export_` from the list of tasks available, then select `task_1` (and not `_import_`); finally, select `example_data_plus_one.csv` as the only dependency for `_export_`.
 
 Once `_export_` has been rigged, this example project is a complete (if exceedingly simple) example of a `pdpp` workflow. The workflow imports a simple `.csv` file, adds one to each number in the file, and exports the resulting modified `.csv` file. `pdpp` workflows can be visualized using the built-in visualization suite like so:
 
@@ -124,7 +124,7 @@ The above command will prompt users for two pieces of information: the output fo
 
 ![](img/dependencies_all.png)
 
-In `pdpp` visualizations, the box-like nodes represent tasks, the nodes with the folded-corners repesent data files, and the nodes with two tabs on the left-hand side represent source code. 
+In `pdpp` visualizations, the box-like nodes represent tasks, the nodes with the folded-corners repesent data files, and the nodes with two tabs on the left-hand side represent source code.
 
 One may execute the entire workflow by using one of the two following commands (both are functionally identical):
 
@@ -148,7 +148,7 @@ When a workflow is run, the `doit` automation suite -- atop which `pdpp` is buil
 -- task_1
 ```
 
-This is because `doit` checks the relative ages of each tasks' inputs and outputs at runtime; if any given task has any outputsOr 'targets,' in `doit` nomenclature.] that are older than one or more of the task's inputs,Or 'dependencies,' in `doit` nomenclature] that task must be re-run. If all of a task's inputs are older than its outputs, the task does not need to be run. This means that a `pdpp`/`doit` pipeline can be run as often as the user desires without running the risk of needlessly wasting time or computing power: tasks will only be re-run if changes to 'upstream' files necessitate it. You can read more about this impressive feature of the `doit` suite [here](https://pydoit.org/tasks.html).  
+This is because `doit` checks the relative ages of each tasks' inputs and outputs at runtime; if any given task has any outputsOr 'targets,' in `doit` nomenclature.] that are older than one or more of the task's inputs,Or 'dependencies,' in `doit` nomenclature] that task must be re-run. If all of a task's inputs are older than its outputs, the task does not need to be run. This means that a `pdpp`/`doit` pipeline can be run as often as the user desires without running the risk of needlessly wasting time or computing power: tasks will only be re-run if changes to 'upstream' files necessitate it. You can read more about this impressive feature of the `doit` suite [here](https://pydoit.org/tasks.html).
 
 
 ## Usage from the Command Line
@@ -170,7 +170,7 @@ Adds a new custom task to a `pdpp` project and launches an interactive rigging s
 
 ### `pdpp sub`
 
-Adds a new sub-project task to a `pdpp` project and launches an interactive rigging session for it (see `pdpp rig` below for more information). Sub-project tasks are distinct `pdpp` projects nested inside the main project -- structurally, they function identically to all other `pdpp` projects. Their dependencies are defined as any local files contained inside their `_import_` directory (which functions as if it were an `input` directory for a task) and their targets are defined as any local files contained inside their `_export_` directory (which functions as if if were an `output` directory for a task). 
+Adds a new sub-project task to a `pdpp` project and launches an interactive rigging session for it (see `pdpp rig` below for more information). Sub-project tasks are distinct `pdpp` projects nested inside the main project -- structurally, they function identically to all other `pdpp` projects. Their dependencies are defined as any local files contained inside their `_import_` directory (which functions as if it were an `input` directory for a task) and their targets are defined as any local files contained inside their `_export_` directory (which functions as if if were an `output` directory for a task).
 
 
 ### `pdpp rig`
@@ -179,7 +179,7 @@ Launches an interactive rigging session for a selected task, which allows users
 
 ### `pdpp run` or `doit`
 
-Runs the project. The `pdpp run` command provides basic functionality; users may pass arguments to the `doit` command that provides a great deal of control and specificity. More information about the `doit` command can be found [here](https://pydoit.org/cmd-run.html). 
+Runs the project. The `pdpp run` command provides basic functionality; users may pass arguments to the `doit` command that provides a great deal of control and specificity. More information about the `doit` command can be found [here](https://pydoit.org/cmd-run.html).
 
 ### `pdpp graph`
 
@@ -196,4 +196,4 @@ Incorporates an already-PDP compliant directory (containing `input`, `output`, a
 
 ### `pdpp enable`
 
-Allows users to toggle tasks 'on' or 'off'; tasks that are 'off' will not be executed when `pdpp run` or `doit` is used. 
+Allows users to toggle tasks 'on' or 'off'; tasks that are 'off' will not be executed when `pdpp run` or `doit` is used.

README.md

@@ -0,0 +1,5 @@
+import doit
+
+from pdpp.automation.task_creator import gen_many_tasks, task_all
+
+doit.run(globals())

dodo.py

@@ -5,10 +5,10 @@ channels:
 dependencies:
   - python
   - pip
-  - Click >= 7.0 
-  - doit >= 0.31.1 
+  - Click >= 7.0
+  - doit >= 0.31.1
   - networkx >= 2.2
   - graphviz >= 0.10.1
   - pydot >= 1.4.1
   - questionary >= 1.0.2
-  - pyyaml >= 5.3
+  - pyyaml >= 5.3

env.yml

@@ -1,8 +1,11 @@
 from pdpp.automation.task_creator import gen_many_tasks, task_all
 
+
 def doit_run():
     import doit
+
     doit.run(globals())
 
-if __name__ == '__main__':
-    doit_run()
+
+if __name__ == "__main__":
+    doit_run()

pdpp/__init__.py

@@ -1,53 +1,83 @@
-from pdpp.tasks.base_task import BaseTask
-from pdpp.automation.mylinker import file_linker, dir_linker
-from typing import List
-from posixpath import join
 import os
+from posixpath import join
+from typing import List
 
+from pdpp.automation.mylinker import dir_linker, file_linker
+from pdpp.tasks.base_task import BaseTask
 
-def make_link_task(task: BaseTask, disabled_list: List[str], final_dep_list: List):
 
+def make_link_task(task: BaseTask, disabled_list: List[str], final_dep_list: List):
     for task_with_dependency, dependency_metadata in task.dep_files.items():
-
-        link_action_list = [] 
+        link_action_list = []
         link_dep_list = []
         link_targ_list = []
 
-        if task_with_dependency not in disabled_list: 
+        if task_with_dependency not in disabled_list:
+            file_link_start = [
+                join(task_with_dependency, dependency_metadata.task_out, f)
+                for f in dependency_metadata.file_list
+            ]
+            file_link_end = [
+                join(task.target_dir, task.IN_DIR, f)
+                for f in dependency_metadata.file_list
+            ]
 
-            file_link_start = [join(task_with_dependency, dependency_metadata.task_out, f) for f in dependency_metadata.file_list]
-            file_link_end = [join(task.target_dir, task.IN_DIR, f) for f in dependency_metadata.file_list]
+            link_action_list.extend(
+                [
+                    (file_linker, [fls, fle])
+                    for fls, fle in list(zip(file_link_start, file_link_end))
+                ]
+            )
 
-            link_action_list.extend([(file_linker, [fls, fle]) for fls, fle in list(zip(file_link_start, file_link_end))])
+            dir_link_start = [
+                join(task_with_dependency, dependency_metadata.task_out, f)
+                for f in dependency_metadata.dir_list
+            ]
+            dir_link_end = [
+                join(task.target_dir, task.IN_DIR, f)
+                for f in dependency_metadata.dir_list
+            ]
 
-            dir_link_start = [join(task_with_dependency, dependency_metadata.task_out, f) for f in dependency_metadata.dir_list]
-            dir_link_end = [join(task.target_dir, task.IN_DIR, f) for f in dependency_metadata.dir_list]
-
-            link_action_list.extend([(dir_linker, [dls, dle]) for dls, dle in list(zip(dir_link_start, dir_link_end))])
+            link_action_list.extend(
+                [
+                    (dir_linker, [dls, dle])
+                    for dls, dle in list(zip(dir_link_start, dir_link_end))
+                ]
+            )
 
             link_dep_list.extend(file_link_start)
             link_targ_list.extend(file_link_end)
 
             for dir_dependency in dependency_metadata.dir_list:
-                path_to_dep_dir = join(dependency_metadata.task_name, dependency_metadata.task_out)
+                path_to_dep_dir = join(
+                    dependency_metadata.task_name, dependency_metadata.task_out
+                )
                 startdir = os.getcwd()
                 os.chdir(path_to_dep_dir)
                 for root, _, filenames in os.walk(dir_dependency):
                     for filename in filenames:
-
-                        subdir_filepath_start = join(dependency_metadata.task_name, dependency_metadata.task_out, root, filename)
+                        subdir_filepath_start = join(
+                            dependency_metadata.task_name,
+                            dependency_metadata.task_out,
+                            root,
+                            filename,
+                        )
                         link_dep_list.append(subdir_filepath_start)
 
-                        subdir_filepath_end = join(task.target_dir, task.IN_DIR, root, filename)
+                        subdir_filepath_end = join(
+                            task.target_dir, task.IN_DIR, root, filename
+                        )
                         link_targ_list.append(subdir_filepath_end)
                 os.chdir(startdir)
 
             final_dep_list.extend(link_targ_list)
 
             yield {
-                'basename': '_task_{}_LINK_TO_{}'.format(task_with_dependency, task.target_dir),
-                'actions': link_action_list,
-                'file_dep': link_dep_list,
-                'targets': link_targ_list,
-                'clean': True,
-            }
+                "basename": "_task_{}_LINK_TO_{}".format(
+                    task_with_dependency, task.target_dir
+                ),
+                "actions": link_action_list,
+                "file_dep": link_dep_list,
+                "targets": link_targ_list,
+                "clean": True,
+            }

pdpp/automation/__init__.py

@@ -1,18 +1,18 @@
 from os import link, remove
-from shutil import rmtree, copytree
+from shutil import copytree, rmtree
 
 
 def file_linker(link_start, link_end):
     try:
         link(link_start, link_end)
     except FileExistsError:
         remove(link_end)
-        link(link_start, link_end)   
+        link(link_start, link_end)
 
 
 def dir_linker(link_start, link_end):
     try:
         copytree(link_start, link_end, copy_function=link)
     except FileExistsError:
         rmtree(link_end)
-        copytree(link_start, link_end, copy_function=link)
+        copytree(link_start, link_end, copy_function=link)