From b03b63351d79b8c3c9906033e9115642cf684659 Mon Sep 17 00:00:00 2001 From: Jeremy Howard Date: Tue, 29 Oct 2024 12:02:22 +1000 Subject: [PATCH] complete refactoring --- execnb/_modidx.py | 1 + execnb/shell.py | 53 +++++++++-------- nbs/02_shell.ipynb | 138 +++++++++++++++++++++------------------------ 3 files changed, 93 insertions(+), 99 deletions(-) diff --git a/execnb/_modidx.py b/execnb/_modidx.py index a3d689e..258215b 100644 --- a/execnb/_modidx.py +++ b/execnb/_modidx.py @@ -48,6 +48,7 @@ 'execnb.shell._out_stream': ('shell.html#_out_stream', 'execnb/shell.py'), 'execnb.shell.exec_nb': ('shell.html#exec_nb', 'execnb/shell.py'), 'execnb.shell.find_output': ('shell.html#find_output', 'execnb/shell.py'), + 'execnb.shell.format_exc': ('shell.html#format_exc', 'execnb/shell.py'), 'execnb.shell.out_error': ('shell.html#out_error', 'execnb/shell.py'), 'execnb.shell.out_exec': ('shell.html#out_exec', 'execnb/shell.py'), 'execnb.shell.out_stream': ('shell.html#out_stream', 'execnb/shell.py')}}} diff --git a/execnb/shell.py b/execnb/shell.py index b268f82..030d715 100644 --- a/execnb/shell.py +++ b/execnb/shell.py @@ -31,9 +31,9 @@ from .nbio import _dict2obj # %% auto 0 -__all__ = ['CaptureShell', 'find_output', 'out_exec', 'out_stream', 'out_error', 'exec_nb', 'SmartCompleter'] +__all__ = ['CaptureShell', 'format_exc', 'find_output', 'out_exec', 'out_stream', 'out_error', 'exec_nb', 'SmartCompleter'] -# %% ../nbs/02_shell.ipynb 7 +# %% ../nbs/02_shell.ipynb 5 class _CustDisplayHook(DisplayHook): def write_output_prompt(self): pass def write_format_data(self, data, md_dict): pass @@ -45,18 +45,17 @@ def __repr__(self: ExecutionInfo): return f'cell: {self.raw_cell}; id: {self.cel @patch def __repr__(self: ExecutionResult): return f'result: {self.result}; err: {self.error_in_exec}; info: <{self.info}>' -# %% ../nbs/02_shell.ipynb 8 +# %% ../nbs/02_shell.ipynb 6 class CaptureShell(InteractiveShell): displayhook_class = _CustDisplayHook - def __init__(self, - path:str|Path=None): # Add `path` to python path - "Execute the IPython/Jupyter source code" + def __init__(self, path:str|Path=None): super().__init__() self.result,self.exc = None,None if path: self.set_path(path) - self.display_formatter.active = False - self.run_cell('%matplotlib inline') # Enable inline plotting + self.display_formatter.active = True + if not IN_NOTEBOOK: InteractiveShell._instance = self + self.run_cell('%matplotlib inline') def run_cell(self, raw_cell, store_history=False, silent=False, shell_futures=True, cell_id=None, stdout=True, stderr=True, display=True): @@ -73,12 +72,17 @@ def set_path(self, path): if path.is_file(): path = path.parent self.run_cell(f"import sys; sys.path.insert(0, '{path.as_posix()}')") -# %% ../nbs/02_shell.ipynb 24 +# %% ../nbs/02_shell.ipynb 22 +def format_exc(e): + "Format exception `e` as a string" + return ''.join(traceback.format_exception(type(e), e, e.__traceback__)) + +# %% ../nbs/02_shell.ipynb 23 def _out_stream(text, name): return dict(name=name, output_type='stream', text=text.splitlines(True)) def _out_exc(e): ename = type(e).__name__ tb = traceback.extract_tb(e.__traceback__)#.format() - return dict(ename=str(ename), evalue=str(e), output_type='error', traceback=tb) + return dict(ename=str(ename), evalue=str(e), output_type='error', traceback=format_exc(e)) def _format_mimedata(k, v): "Format mime-type keyed data consistently with Jupyter" @@ -103,7 +107,7 @@ def _out_nb(o, fmt): res.append(_mk_out(*fmt.format(r), 'execute_result')) return res -# %% ../nbs/02_shell.ipynb 25 +# %% ../nbs/02_shell.ipynb 24 @patch def run(self:CaptureShell, code:str, # Python/IPython code to run @@ -115,7 +119,7 @@ def run(self:CaptureShell, self.exc = res.exception return _out_nb(res, self.display_formatter) -# %% ../nbs/02_shell.ipynb 39 +# %% ../nbs/02_shell.ipynb 38 @patch def cell(self:CaptureShell, cell, stdout=True, stderr=True): "Run `cell`, skipping if not code, and store outputs back in cell" @@ -127,32 +131,32 @@ def cell(self:CaptureShell, cell, stdout=True, stderr=True): for o in outs: if 'execution_count' in o: cell['execution_count'] = o['execution_count'] -# %% ../nbs/02_shell.ipynb 42 +# %% ../nbs/02_shell.ipynb 41 def find_output(outp, # Output from `run` ot='execute_result' # Output_type to find ): "Find first output of type `ot` in `CaptureShell.run` output" return first(o for o in outp if o['output_type']==ot) -# %% ../nbs/02_shell.ipynb 45 +# %% ../nbs/02_shell.ipynb 44 def out_exec(outp): "Get data from execution result in `outp`." out = find_output(outp) if out: return '\n'.join(first(out['data'].values())) -# %% ../nbs/02_shell.ipynb 47 +# %% ../nbs/02_shell.ipynb 46 def out_stream(outp): "Get text from stream in `outp`." out = find_output(outp, 'stream') if out: return ('\n'.join(out['text'])).strip() -# %% ../nbs/02_shell.ipynb 49 +# %% ../nbs/02_shell.ipynb 48 def out_error(outp): "Get traceback from error in `outp`." out = find_output(outp, 'error') if out: return '\n'.join(out['traceback']) -# %% ../nbs/02_shell.ipynb 51 +# %% ../nbs/02_shell.ipynb 50 def _false(o): return False @patch @@ -172,7 +176,7 @@ def run_all(self:CaptureShell, postproc(cell) if self.exc and exc_stop: raise self.exc from None -# %% ../nbs/02_shell.ipynb 65 +# %% ../nbs/02_shell.ipynb 64 @patch def execute(self:CaptureShell, src:str|Path, # Notebook path to read from @@ -193,7 +197,7 @@ def execute(self:CaptureShell, inject_code=inject_code, inject_idx=inject_idx) if dest: write_nb(nb, dest) -# %% ../nbs/02_shell.ipynb 69 +# %% ../nbs/02_shell.ipynb 68 @patch def prettytb(self:CaptureShell, fname:str|Path=None): # filename to print alongside the traceback @@ -201,11 +205,11 @@ def prettytb(self:CaptureShell, fname = fname if fname else self._fname _fence = '='*75 cell_intro_str = f"While Executing Cell #{self._cell_idx}:" if self._cell_idx else "While Executing:" - cell_str = f"\n{cell_intro_str}\n{self.exc}" + cell_str = f"\n{cell_intro_str}\n{format_exc(self.exc)}" fname_str = f' in {fname}' if fname else '' return f"{type(self.exc).__name__}{fname_str}:\n{_fence}\n{cell_str}\n" -# %% ../nbs/02_shell.ipynb 88 +# %% ../nbs/02_shell.ipynb 87 @call_parse def exec_nb( src:str, # Notebook path to read from @@ -219,13 +223,12 @@ def exec_nb( CaptureShell().execute(src, dest, exc_stop=exc_stop, inject_code=inject_code, inject_path=inject_path, inject_idx=inject_idx) -# %% ../nbs/02_shell.ipynb 91 +# %% ../nbs/02_shell.ipynb 90 class SmartCompleter(IPCompleter): - def __init__(self, shell, namespace=None): + def __init__(self, shell, namespace=None, jedi=False): if namespace is None: namespace = shell.user_ns super().__init__(shell, namespace) - self.use_jedi = False - + self.use_jedi = jedi sdisp = StrDispatch() self.custom_completers = sdisp import_disp = CommandChainDispatcher() diff --git a/nbs/02_shell.ipynb b/nbs/02_shell.ipynb index 117c7c1..51550c5 100644 --- a/nbs/02_shell.ipynb +++ b/nbs/02_shell.ipynb @@ -73,61 +73,6 @@ "## CaptureShell -" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class SmartCompleter(IPCompleter):\n", - " def __init__(self, shell, namespace=None):\n", - " if namespace is None: namespace = shell.user_ns\n", - " super().__init__(shell, namespace)\n", - " self.use_jedi = False\n", - "\n", - " sdisp = StrDispatch()\n", - " self.custom_completers = sdisp\n", - " import_disp = CommandChainDispatcher()\n", - " import_disp.add(types.MethodType(module_completer, shell))\n", - " sdisp.add_s('import', import_disp)\n", - " sdisp.add_s('from', import_disp)\n", - "\n", - " def __call__(self, c):\n", - " if not c: return []\n", - " with provisionalcompleter():\n", - " return [o.text.rpartition('.')[-1]\n", - " for o in self.completions(c, len(c))\n", - " if o.type=='']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cc = SmartCompleter(get_ipython())\n", - "\n", - "def test_set(a,b): return test_eq(set(a), set(b))\n", - "\n", - "class _f:\n", - " def __init__(self): self.bar,self.baz,self.room = 0,0,0\n", - "\n", - "foo = _f()\n", - "\n", - "test_set(cc(\"b\"), ['bin', 'bool', 'break', 'breakpoint', 'bytearray', 'bytes'])\n", - "test_set(cc(\"foo.b\"), ['bar', 'baz'])\n", - "test_set(cc(\"x=1; x = foo.b\"), ['bar', 'baz'])\n", - "test_set(cc(\"ab\"), ['abs'])\n", - "test_set(cc(\"b = ab\"), ['abs'])\n", - "test_set(cc(\"\"), [])\n", - "test_set(cc(\"foo.\"), ['bar', 'baz', 'room'])\n", - "test_set(cc(\"nonexistent.b\"), [])\n", - "test_set(cc(\"foo.nonexistent.b\"), [])\n", - "test_set(cc(\"import ab\"), ['abc'])\n", - "test_set(cc(\"from abc import AB\"), ['ABC', 'ABCMeta'])" - ] - }, { "cell_type": "code", "execution_count": null, @@ -157,14 +102,13 @@ "class CaptureShell(InteractiveShell):\n", " displayhook_class = _CustDisplayHook\n", "\n", - " def __init__(self,\n", - " path:str|Path=None): # Add `path` to python path\n", - " \"Execute the IPython/Jupyter source code\"\n", + " def __init__(self, path:str|Path=None):\n", " super().__init__()\n", " self.result,self.exc = None,None\n", " if path: self.set_path(path)\n", - " self.display_formatter.active = False\n", - " self.run_cell('%matplotlib inline') # Enable inline plotting\n", + " self.display_formatter.active = True\n", + " if not IN_NOTEBOOK: InteractiveShell._instance = self\n", + " self.run_cell('%matplotlib inline')\n", "\n", " def run_cell(self, raw_cell, store_history=False, silent=False, shell_futures=True, cell_id=None,\n", " stdout=True, stderr=True, display=True):\n", @@ -640,6 +584,18 @@ "### Cells" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def format_exc(e):\n", + " \"Format exception `e` as a string\"\n", + " return ''.join(traceback.format_exception(type(e), e, e.__traceback__))" + ] + }, { "cell_type": "code", "execution_count": null, @@ -651,7 +607,7 @@ "def _out_exc(e):\n", " ename = type(e).__name__\n", " tb = traceback.extract_tb(e.__traceback__)#.format()\n", - " return dict(ename=str(ename), evalue=str(e), output_type='error', traceback=tb)\n", + " return dict(ename=str(ename), evalue=str(e), output_type='error', traceback=format_exc(e))\n", "\n", "def _format_mimedata(k, v):\n", " \"Format mime-type keyed data consistently with Jupyter\"\n", @@ -743,7 +699,7 @@ "[{'name': 'stdout',\n", " 'output_type': 'stream',\n", " 'text': ['CPU times: user 1 us, sys: 0 ns, total: 1 us\\n',\n", - " 'Wall time: 3.1 us\\n']},\n", + " 'Wall time: 2.86 us\\n']},\n", " {'data': {'text/plain': ['2']},\n", " 'metadata': {},\n", " 'output_type': 'execute_result'}]" @@ -870,8 +826,7 @@ " {'ename': 'Exception',\n", " 'evalue': 'Oops',\n", " 'output_type': 'error',\n", - " 'traceback': [,\n", - " , line 1 in >]}]" + " 'traceback': 'Traceback (most recent call last):\\n File \"/Users/jhoward/miniconda3/lib/python3.11/site-packages/IPython/core/interactiveshell.py\", line 3553, in run_code\\n exec(code_obj, self.user_global_ns, self.user_ns)\\n File \"\", line 1, in \\n raise Exception(\"Oops\")\\nException: Oops\\n'}]" ] }, "execution_count": null, @@ -1249,8 +1204,7 @@ " {'ename': 'Exception',\n", " 'evalue': 'Oopsie!',\n", " 'output_type': 'error',\n", - " 'traceback': [,\n", - " , line 1 in >]}]" + " 'traceback': 'Traceback (most recent call last):\\n File \"/Users/jhoward/miniconda3/lib/python3.11/site-packages/IPython/core/interactiveshell.py\", line 3553, in run_code\\n exec(code_obj, self.user_global_ns, self.user_ns)\\n File \"\", line 1, in \\n raise Exception(\"Oopsie!\")\\nException: Oopsie!\\n'}]" ] }, "execution_count": null, @@ -1278,7 +1232,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "got exception: 'Exception' object is not subscriptable\n" + "got exception: Oopsie!\n" ] } ], @@ -1400,7 +1354,15 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[{'name': 'stdout', 'output_type': 'stream', 'text': ['1\\n']}, {'data': {'text/plain': ['2']}, 'metadata': {}, 'output_type': 'execute_result'}]\n" + ] + } + ], "source": [ "s = CaptureShell()\n", "try:\n", @@ -1433,7 +1395,7 @@ " fname = fname if fname else self._fname\n", " _fence = '='*75\n", " cell_intro_str = f\"While Executing Cell #{self._cell_idx}:\" if self._cell_idx else \"While Executing:\"\n", - " cell_str = f\"\\n{cell_intro_str}\\n{self.exc}\"\n", + " cell_str = f\"\\n{cell_intro_str}\\n{format_exc(self.exc)}\"\n", " fname_str = f' in {fname}' if fname else ''\n", " return f\"{type(self.exc).__name__}{fname_str}:\\n{_fence}\\n{cell_str}\\n\"" ] @@ -1449,7 +1411,36 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "AssertionError in ../tests/error.ipynb:\n", + "===========================================================================\n", + "\n", + "While Executing Cell #2:\n", + "Traceback (most recent call last):\n", + " File \"/var/folders/ss/34z569j921v58v8n1n_8z7h40000gn/T/ipykernel_85617/1421292703.py\", line 3, in \n", + " s.execute('../tests/error.ipynb', exc_stop=True)\n", + " File \"/var/folders/ss/34z569j921v58v8n1n_8z7h40000gn/T/ipykernel_85617/2032771714.py\", line 18, in execute\n", + " self.run_all(nb, exc_stop=exc_stop, preproc=preproc, postproc=postproc,\n", + " File \"/var/folders/ss/34z569j921v58v8n1n_8z7h40000gn/T/ipykernel_85617/1738204067.py\", line 19, in run_all\n", + " if self.exc and exc_stop: raise self.exc from None\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/Users/jhoward/miniconda3/lib/python3.11/site-packages/IPython/core/interactiveshell.py\", line 3553, in run_code\n", + " exec(code_obj, self.user_global_ns, self.user_ns)\n", + " File \"\", line 3, in \n", + " foo()\n", + " File \"/Users/jhoward/Documents/GitHub/execnb/tests/err.py\", line 2, in foo\n", + " assert 13 == 98\n", + " ^^^^^^^^\n", + "AssertionError\n", + "\n", + "\n" + ] + } + ], "source": [ "s = CaptureShell()\n", "try:\n", @@ -1698,7 +1689,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Completions" + "## Completions -" ] }, { @@ -1709,11 +1700,10 @@ "source": [ "#| export\n", "class SmartCompleter(IPCompleter):\n", - " def __init__(self, shell, namespace=None):\n", + " def __init__(self, shell, namespace=None, jedi=False):\n", " if namespace is None: namespace = shell.user_ns\n", " super().__init__(shell, namespace)\n", - " self.use_jedi = False\n", - "\n", + " self.use_jedi = jedi\n", " sdisp = StrDispatch()\n", " self.custom_completers = sdisp\n", " import_disp = CommandChainDispatcher()\n",