add %sqlcmd explore (#420)

* initial version added

* sql pagination added

* test fixed

* sort columns added

* changelog updated

* Allows to add multiple tables to a single notebook

* tests added

* testing on binder

* binder test

* _get_binder_hub_url added

* webscoket connection added

* table_widget added

* rebase

* lint

* widgets directory name changed

* imports

* short import removed

* __init__ added

* import fixed

* lint

* import

* telemetry added

* pagination bug on Jupyter nb fixed

* docs added

* clean

* lint

* tests added

* path fixed

* path changed

* lint

* table_widget js and css files added

* empty commit invoke CI

* apt.txt added with nodejs

* apt removed. doc updated

* apt removed

* docs fixed

* apt.txt added

* docs

* cleaned

* code review fixes

* code review fixes

* lint

* lint

* psutil added

* lint

* link to binder fixed

* table_widget.jpg removed

CHANGELOG.md

@@ -1,6 +1,7 @@
 # CHANGELOG
 
 ## 0.7.6dev
+* [Feature] Add `%sqlcmd explore` to explore tables interactively (#330)
 
 * [Fix] Fix error when checking if custom connection was PEP 249 Compliant (#517)
 * [Feature] Support for printing capture variables using `=<<` syntax (by [@jorisroovers](https://github.com/jorisroovers))

MANIFEST.in

@@ -1,3 +1,5 @@
 include README.rst
 include NEWS.rst
 include LICENSE
+include src/sql/widgets/table_widget/css/*
+include src/sql/widgets/table_widget/js/*

doc/_toc.yml

@@ -17,6 +17,7 @@ parts:
     - file: user-guide/interactive
     - file: user-guide/data-profiling
     - file: user-guide/ggplot
+    - file: user-guide/table_explorer
     - file: user-guide/FAQ
 
   - caption: Integrations

doc/conf.py

@@ -66,9 +66,11 @@
     "repository_branch": repository_branch,
     "analytics": {"google_analytics_id": "G-JBZ8NNQSLN"},
     "home_page_in_toc": True,
-    "announcement": ("To launch a tutorial, click on the 🚀 button "
-                     "below! Join us on "
-                     "<a href='https://ploomber.io/community/'>Slack!</a>"),
+    "announcement": (
+        "To launch a tutorial, click on the 🚀 button "
+        "below! Join us on "
+        "<a href='https://ploomber.io/community/'>Slack!</a>"
+    ),
     "use_repository_button": True,
     "use_edit_page_button": False,
     "use_issues_button": True,
@@ -106,18 +108,17 @@
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory.
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
 # Load custom stylesheets to support Algolia search.
-html_css_files = [
-    'algolia.css',
-    'https://cdn.jsdelivr.net/npm/@docsearch/css@3'
-]
+html_css_files = ["algolia.css", "https://cdn.jsdelivr.net/npm/@docsearch/css@3"]
 
 # Load custom javascript to support Algolia search. Note that the sequence
 # defined below (external first) is intentional!
 html_js_files = [
-    ('https://cdn.jsdelivr.net/npm/@docsearch/[email protected]/dist/umd/index.js',
-     {'defer': 'defer'}),
-    ('algolia.js', {'defer': 'defer'})
+    (
+        "https://cdn.jsdelivr.net/npm/@docsearch/[email protected]/dist/umd/index.js",
+        {"defer": "defer"},
+    ),
+    ("algolia.js", {"defer": "defer"}),
 ]

doc/user-guide/table_explorer.md

@@ -0,0 +1,71 @@
+---
+jupytext:
+  notebook_metadata_filter: myst
+  text_representation:
+    extension: .md
+    format_name: myst
+    format_version: 0.13
+    jupytext_version: 1.14.5
+kernelspec:
+  display_name: Python 3 (ipykernel)
+  language: python
+  name: python3
+myst:
+  html_meta:
+    description lang=en: Templatize SQL queries in Jupyter via JupySQL
+    keywords: jupyter, sql, jupysql, jinja
+    property=og:locale: en_US
+---
+
+# Table Explorer
+
+
+```{versionadded} 0.7.6
+~~~
+pip install jupysql --upgrade
+~~~
+```
+
+In this guide, we demonstrate how to use JupySQL's table explorer to visualize SQL tables in HTML format and interact with them efficiently. By running SQL queries in the background instead of loading the data into memory, we minimize the resource consumption and processing time required for handling large datasets, making the interaction with the SQL tables faster and more streamlined.
+
+```{note}
+If you are using JupyterLab or Binder, please ensure that you have installed the latest version of the JupySQL plugin by running the following command: `pip install jupysql-plugin --upgrade`.
+```
+
+Let's start by preparing our dataset. We'll be using the [NYC taxi dataset](https://www.nyc.gov/site/tlc/about/tlc-trip-record-data.page).
+
+## Download the data
+
+```{code-cell} ipython3
+from pathlib import Path
+from urllib.request import urlretrieve
+
+url = "https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-01.parquet"
+
+if not Path("yellow_tripdata_2021-01.parquet").is_file():
+    urlretrieve(url, "yellow_tripdata_2021.parquet")
+```
+
+## Set connection
+
+After our dataset is ready, we should set our connection.
+
+For this demonstration, we'll be using the `DuckDB` connection.
+
+```{code-cell} ipython3
+%load_ext sql
+%sql duckdb://
+```
+
+## Create the table
+
+To create the table, use the `explore` attribute and specify the name of the table that was just downloaded.
+
+```{code-cell} ipython3
+:tags: []
+
+%sqlcmd explore --table "yellow_tripdata_2021.parquet"
+```
+
+
+See interactive and live example on [Binder](https://binder.ploomber.io/v2/gh/ploomber/jupysql/master?urlpath=lab/tree/doc/user-guide/table_explorer.md).

setup.py

@@ -28,6 +28,7 @@
     "sqlglot>=11.3.7",
     "ploomber-core>=0.2.7",
     'importlib-metadata;python_version<"3.8"',
+    "psutil",
 ]
 
 DEV = [
@@ -47,6 +48,8 @@
     "black",
     # for %%sql --interact
     "ipywidgets",
+    # for running tests for %sqlcmd explore --table
+    "js2py",
 ]
 
 # dependencies for running integration tests

src/sql/magic.py

@@ -41,6 +41,7 @@
 
 from sql.telemetry import telemetry
 
+
 SUPPORT_INTERACTIVE_WIDGETS = ["Checkbox", "Text", "IntSlider", ""]
 
 

src/sql/magic_cmd.py

@@ -21,6 +21,9 @@
 from sql.util import sanitize_identifier
 from sql import exceptions
 
+from sql.widgets import TableWidget
+from IPython.display import display
+
 
 class CmdParser(argparse.ArgumentParser):
     def exit(self, status=0, message=None):
@@ -47,7 +50,7 @@ def _validate_execute_inputs(self, line):
         # We rely on SQLAlchemy when inspecting tables
         util.support_only_sql_alchemy_connection("%sqlcmd")
 
-        AVAILABLE_SQLCMD_COMMANDS = ["tables", "columns", "test", "profile"]
+        AVAILABLE_SQLCMD_COMMANDS = ["tables", "columns", "test", "profile", "explore"]
 
         if line == "":
             raise exceptions.UsageError(
@@ -210,6 +213,16 @@ def execute(self, cmd_name="", others="", cell="", local_ns=None):
 
             return report
 
+        elif cmd_name == "explore":
+            parser = CmdParser()
+            parser.add_argument(
+                "-t", "--table", type=str, help="Table name", required=True
+            )
+            args = parser.parse_args(others)
+
+            table_widget = TableWidget(args.table)
+            display(table_widget)
+
 
 def return_test_results(args, conn, query):
     try:

src/sql/util.py

@@ -1,8 +1,10 @@
+import sql
 from sql import inspect
 import difflib
 from sql.connection import Connection
 from sql.store import store
 from sql import exceptions
+import json
 
 SINGLE_QUOTE = "'"
 DOUBLE_QUOTE = '"'
@@ -250,3 +252,56 @@ def support_only_sql_alchemy_connection(command):
     """
     if Connection.is_custom_connection():
         raise exceptions.RuntimeError(f"{command} is not supported for a custom engine")
+
+
+def fetch_sql_with_pagination(
+    table, offset, n_rows, sort_column=None, sort_order=None
+) -> tuple:
+    """
+    Returns next n_rows and columns from table starting at the offset
+
+    Parameters
+    ----------
+    table : str
+        Table name
+
+    offset : int
+        Specifies the number of rows to skip before
+        it starts to return rows from the query expression.
+
+    n_rows : int
+        Number of rows to return.
+
+    sort_column : str, default None
+        Sort by column
+
+    sort_order : 'DESC' or 'ASC', default None
+        Order list
+    """
+    is_table_exists(table)
+
+    order_by = "" if not sort_column else f"ORDER BY {sort_column} {sort_order}"
+
+    query = f"""
+    SELECT * FROM {table} {order_by}
+    OFFSET {offset} ROWS FETCH NEXT {n_rows} ROWS ONLY"""
+
+    rows = Connection.current.execute(query).fetchall()
+
+    columns = sql.run.raw_run(
+        Connection.current, f"SELECT * FROM {table} WHERE 1=0"
+    ).keys()
+
+    return rows, columns
+
+
+def parse_sql_results_to_json(rows, columns) -> str:
+    """
+    Serializes sql rows to a JSON formatted ``str``
+    """
+    dicts = [dict(zip(list(columns), row)) for row in rows]
+    rows_json = json.dumps(dicts, indent=4, sort_keys=True, default=str).replace(
+        "null", '"None"'
+    )
+
+    return rows_json

src/sql/widgets/__init__.py

@@ -0,0 +1,3 @@
+from sql.widgets.table_widget.table_widget import TableWidget
+
+__all__ = ["TableWidget"]

src/sql/widgets/table_widget/css/tableWidget.css

@@ -0,0 +1,23 @@
+.sort-button {
+    background: none;
+    border: none;
+}
+
+.sort-button.selected {
+    background: #efefef;
+    border: 1px solid #767676;
+}
+
+.pages-buttons button.selected {
+    background: #efefef;
+    border: 1px solid #767676;
+    border-radius: 2px;
+}
+.pages-buttons button {
+    background: none;
+    border: none;
+    padding: 0 10px;
+}
+.jupysql-table-widget {
+    display: inline;
+}