Merge pull request #1493 from saulpw/develop

Prep for v2.10 release

Author: saulpw

.circleci/config.yml

@@ -0,0 +1,54 @@
+name: visidata-ci-build
+on:
+  pull_request:
+    branches:
+      - stable
+      - develop
+  push:
+    branches:
+    - stable
+    - develop
+
+jobs:
+  run-tests:
+
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8, 3.9, "3.10"]
+
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.pythonversion }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    
+    - name: Create .visidatarc
+      run: touch ~/.visidatarc
+
+    - name: Set up Environment
+      run: |
+        export PYTHONPATH=~/visidata:~/visidata/visidata
+        locale
+
+    - name: Install Base VisiData
+      run: pip3 install .
+
+    - name: Ensure VisiData works with Base Dependencies
+      run: vd --version
+
+    - name: Install optional dependencies
+      run: |
+        pip3 install -r requirements.txt
+        pip3 install pytest
+
+    - name: Run test_commands
+      run: |
+        pytest -sv visidata/tests/test_commands.py
+
+    - name: Run Cmdlog Tests
+      run: dev/test.sh
+      shell: bash
+

.envrc

@@ -12,4 +12,5 @@ visidata.egg-info/
 tags
 /tests/log/
 # Mac files
-.DS_Store
+.DS_Store
+.direnv

.github/workflows/main.yml

@@ -1,5 +1,72 @@
 # VisiData version history
 
+# v2.10 (2022-08-28)
+
+- [plugins] load all entry points in `visidata.plugins` group before config load
+  - add entry_points={'visidata.plugins': 'foo=foo'} to plugin load plugin automatically when launching VisiData
+
+- [deps] require `importlib_metadata` >= 3.6
+  - following https://github.com/pypa/twine/pull/732
+
+## Improvements
+
+- [draw] redraw only every 100 ms if any keys pending (requested by @ansoncg #1459)
+- [IndexSheet] shown name is only final name component
+- [loaders html] add table of all links on page (requested by @dufferzafar #1424)
+- [loaders html] `open-row` on **LinksSheet** to open url (requested by @dufferzafar #1424)
+- [options] add `options.http_req_*` to send headers/etc to requests (requested by @daviewales #1446)
+- [options pyobj] add `options.fmt_expand_dict` and `options.fmt_expand_list` for formatting expanded
+    list and dict column names (requested by @joe-opensrc #1457)
+- [threads-sheet] add `z Ctrl+T` (`threads-sheet`) to open **ThreadsSheet** for current sheet
+- [threads-sheet] add `g Ctrl+C` (`cancel-all`) to **ThreadsSheet**
+- [zsh] add scripts for zsh completion (PR by @Freed-Wu #1455)
+  - tutorial: https://visidata.org/docs/shell/
+
+## Bugfixes
+
+- [addcol-] set cursor at added column for `addcol-new`/`addcol-bulk` (reported by @jsvine #1445)
+- [cmdlog] `Ctrl+S` from a **CommandLog** now defaults to `.vdj` (reported by @jsvine #1443)
+- [display] format entire string for undetermined width (reported by and fixed by @jsvine #1442)
+- [formatter] fix len format strings
+- [LastInputsSheet] catch other exceptions during reload
+- [loader npz] fix .npz loader (reported by @Shahin-rmz #1440)
+- [loader geojson] fix plotting and saving geojson files (fixed by @mwayne #1477)
+- [loader geojson] improve feature property manipulation (fixed by @mwayne #1477)
+- [menu] upon menu item keypress, move to item (reported by @reagle #1470)
+- [menu] fix `ALT+<keystroke>` navigation while within menu (reported by @reagle #1470)
+  - now requires two `ESC` to exit
+- [open] allow binary files from archives (reported by @frosencrantz #1430)
+- [save] do not save unknown filetype as `save_filetype`
+- [save visidatarc] only save rows on **OptionsSheet** to visidatarc
+- [sheets] fix name reconstruction for files with multiple  and no suffixes (#1450)
+- [sheets] do not include empty name parts in sheet name
+- [unzip-http] **FreqTableSheet** `open-row` now loads links (reported by @frosencrantz #1458)
+- [zip] use correct rowdef in extract (reported by @frosencrantz #1430)
+- [zip] do not create directory for extract
+
+## snippets
+
+- add snippets/scrolloff.py which mimics vim's scrollof context lines (requested by @gennaro-tedesco #1441)
+
+## vdplus
+
+- `open-memusage` was moved to vdplus
+
+## API
+
+- add InferColumnsSheet
+  - it infers the columns and their types from the rows it gets which are dicts
+  - used by json, npy loader
+- add vd.printout and vd.printerr for builtins.print to stdout and stderr
+- add `vd.view()`
+- fix Extensible.init() to work with classes with no `__init__`
+- add `Sheet.sidebar` and `Sheet.sidebar_title` properties
+
+## Deprecated
+
+- remove VisiDataSheet
+- remove vdmenu
+
 # v2.9.1 (2022-07-21)
 
 - [unzip-http] move urllib3 to optional dependencies

.gitignore

@@ -1,5 +1,5 @@
 
-# VisiData v2.9.1 [![twitter @VisiData][1.1]][1] [![CircleCI](https://circleci.com/gh/saulpw/visidata/tree/stable.svg?style=svg)](https://circleci.com/gh/saulpw/visidata/tree/stable) [![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/saulpw/visidata)
+# VisiData v2.10 [![twitter @VisiData][1.1]][1] [![CircleCI](https://circleci.com/gh/saulpw/visidata/tree/stable.svg?style=svg)](https://circleci.com/gh/saulpw/visidata/tree/stable) [![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/saulpw/visidata)
 
 A terminal interface for exploring and arranging tabular data.
 

CHANGELOG.md

@@ -16,10 +16,6 @@
     - numerical, boolean and string option
     - sheet-specific and global
     - motd_url
-9. using the pandas loader
-    - select
-    - edit
-    - frequency table
 10. large dataset (311)
 12. Options
     - local + global options should be set appropriately
@@ -87,10 +83,12 @@
 17. Anything new in this release (should it have its own automated test?)
 18. `edit-cell` and then `Ctrl+O` to launch editor.
 19. Save to a non-existent format.
-    - Saves to tsv by default
+    - Saves to save-filetype by default
+    - If save-filetype is a non-existent format, blocks
     - Save to it a second time, asks for confirmation, if confirm_overwrite=True
 20. Save multiple sheets to a single non-embeddable format
     - save name makes sense
     - fails if not offered a directory
     - succeeds if offered a directory
 21. Test macro-record.
+22. Test `open-row` on an html link: https://hls.gsfc.nasa.gov/data/

README.md

@@ -20,7 +20,7 @@ for i in $TESTS ; do
     outbase=${i##tests/}
     if [ "${i%-nosave.vd*}-nosave" == "${i%.vd*}" ];
     then
-        PYTHONPATH=. bin/vd --play "$i" --batch --config tests/.visidatarc --visidata-dir tests/.visidata
+        echo "$1"
     else
         for goldfn in tests/golden/${outbase%.vd*}.*; do
             PYTHONPATH=. bin/vd --confirm-overwrite=False --play "$i" --batch --output "$goldfn" --config tests/.visidatarc --visidata-dir tests/.visidata

dev/checklists/manual-tests.md

@@ -0,0 +1,19 @@
+#compdef vd visidata
+
+(( $+functions[_vd_files] )) ||
+_vd_files () {
+  case $PREFIX in
+    (*) _files $* ;;
+  esac
+  case $PREFIX in
+    (+) _message -e 'toplevel:subsheet:col:row' ;;
+    (+<1->) _message -e 'toplevel' ;;
+    (+<1->:) _message -e 'subsheet' ;;
+    (+<1->:<1->:) _message -e 'col' ;;
+    (+<1->:<1->:<1->:) _message -e 'row' ;;
+  esac
+}
+
+_arguments -S \
+    {{flags}} \
+  '(-p --play -w --replay-wait -b --batch -o --output --replay-movement)*:file:_vd_files'

dev/test.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+import os
+from os.path import dirname as dirn
+import sys
+import re
+
+sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
+from visidata import vd
+from visidata.main import option_aliases
+
+ZSH_COMPLETION_FILE = "_visidata"
+ZSH_COMPLETION_TEMPLATE = "dev/zsh-completion.in"
+pat_class = re.compile("'(.*)'")
+pat_select = re.compile("^\([^)]*\)")
+
+
+def generate_completion(opt):
+    prefix = "--" + opt.name
+    shortnames = [key for key, value in option_aliases.items() if value[0] == opt.name]
+    if len(shortnames):
+        if len(shortnames[0]) == 1:
+            shortname = "-" + shortnames[0]
+        else:
+            shortname = "--" + shortnames[0]
+        prefix = "{" + f"{shortname},{prefix}" + "}"
+    if isinstance(opt.value, bool):
+        completion = ""
+    else:
+        completion = ":" + pat_class.findall(str(opt.value.__class__))[0]
+    if opt.name in ["play", "output", "visidata_dir", "config"]:
+        completion += ":_files"
+    elif opt.name in ["plugins_url", "motd_url"]:
+        completion += ":_urls"
+    helpstr = opt.helpstr.replace("[", "\\[").replace("]", "\\]")
+    selections = pat_select.findall(helpstr)
+    if len(selections):
+        completion += f":{selections[0].replace('/', ' ')}"
+    # TODO: use `zstyle ':completion:*' extra-verbose true`
+    # to control the display of default value
+    helpstr = helpstr + f" (default: {opt.value})"
+    return f"{prefix}'[{helpstr}]{completion}'"
+
+
+flags = [generate_completion(vd._options[opt]["default"]) for opt in vd._options]
+
+with open(ZSH_COMPLETION_TEMPLATE) as f:
+    template = f.read()
+
+template = template.replace("{{flags}}", " \\\n    ".join(flags))
+
+with open(ZSH_COMPLETION_FILE, "w") as f:
+    f.write(template)