fix: Address warnings, error on malformed TSV contents (#2240)

* fix: Opt in to pandas 3.0 behaviors, when possible

* fix: Address warning identified with -Werror

* fix: Raise error on warnings in custom fence

* fix: Bad tabs in TSV examples

Author: effigies

pyproject.toml

@@ -5,7 +5,7 @@ dependencies = [
   "bidsschematools[render]",
   "mkdocs>=1.1.0",
   "mkdocs-material>=5.4",
-  "pymdown-extensions>=7.0.0",
+  "pymdown-extensions>=9.2.0",
   "mkdocs-branchcustomization-plugin~=0.1.3",
   "mkdocs-macros-plugin",
   "mkdocs-redirects",
@@ -57,3 +57,7 @@ replace = """NEXT_VERSION_HERE
 - '{new_version}'
 - """
 include_bumps = ['pre_label']
+
+[[tool.mypy.overrides]]
+module = ["pymdownx.*"]
+ignore_missing_imports = true

src/derivatives/imaging.md

@@ -548,9 +548,9 @@ standard BIDS labels:
 
 ```tsv
 index	name	abbreviation
-137	pars	opercularis	IFGop
-138	pars	triangularis	IFGtr
-139	pars	orbitalis	IFGor
+137	pars opercularis	IFGop
+138	pars triangularis	IFGtr
+139	pars orbitalis	IFGor
 ```
 
 <!-- Link Definitions -->

src/modality-specific-files/microscopy.md

@@ -420,8 +420,8 @@ Additional optional columns MAY be used to describe other subjects' attributes.
 `participants.tsv` example:
 ```tsv
 participant_id	species	strain	strain_rrid
-sub-01	mus	musculus	C57BL/6J	RRID:IMSR_JAX:000664
-sub-02	mus	musculus	C57BL/6J	RRID:IMSR_JAX:000664
+sub-01	mus musculus	C57BL/6J	RRID:IMSR_JAX:000664
+sub-02	mus musculus	C57BL/6J	RRID:IMSR_JAX:000664
 ```
 
 `participants.json` example:

tools/schemacode/pyproject.toml

@@ -43,6 +43,7 @@ render = [
   "tabulate",
   "pandas",
   "markdown-it-py",
+  "pymdown-extensions>=9.2",
 ]
 tests = [
   "bidsschematools[validation,render,expressions]",

tools/schemacode/src/bidsschematools/render/tsv.py

@@ -7,7 +7,12 @@
 from markdown_it import MarkdownIt
 from tabulate import tabulate
 
+from ..utils import filter_warnings, in_context
+from .utils import propagate_fence_exception
 
+
+@in_context(filter_warnings(["error"]))
+@propagate_fence_exception
 def fence(
     source: str,
     language: str,
@@ -27,32 +32,26 @@ def fence(
 
     classes[:0] = ["tsv-table", "index" if linenums else "noindex"]
 
-    try:
-        df = pd.read_csv(
-            io.StringIO(source),
-            sep="\t",
-            dtype=str,
-            index_col=False,
-            keep_default_na=False,
-            header=None if "noheader" in attrs else "infer",
-        )
-        md_table = tabulate(
-            df,  # type: ignore
-            tablefmt="github",
-            showindex=linenums,
-            headers="keys",
-            numalign="right",
-        )
-        html = MarkdownIt("commonmark").enable("table").render(md_table)
-        if "noheader" in attrs:
-            html = re.sub("<thead>.+</thead>", "", html, flags=re.DOTALL)
-
-        html = html.replace("<table>", f'<table class="{" ".join(classes)}">')
-
-        # Remove newlines from HTML to prevent copy-paste from inserting spaces
-        return html.replace("\n", "")
-    except Exception:
-        import traceback
-
-        exc = traceback.format_exc()
-        return f"<pre>{exc}</pre>"
+    df = pd.read_csv(
+        io.StringIO(source),
+        sep="\t",
+        dtype=str,
+        index_col=False,
+        keep_default_na=False,
+        header=None if "noheader" in attrs else "infer",
+    )
+    md_table = tabulate(
+        df,  # type: ignore
+        tablefmt="github",
+        showindex=linenums,
+        headers="keys",
+        numalign="right",
+    )
+    html = MarkdownIt("commonmark").enable("table").render(md_table)
+    if "noheader" in attrs:
+        html = re.sub("<thead>.+</thead>", "", html, flags=re.DOTALL)
+
+    html = html.replace("<table>", f'<table class="{" ".join(classes)}">')
+
+    # Remove newlines from HTML to prevent copy-paste from inserting spaces
+    return html.replace("\n", "")

tools/schemacode/src/bidsschematools/render/utils.py

@@ -3,6 +3,47 @@
 import math
 import posixpath
 import re
+from functools import cache, wraps
+
+import pandas as pd
+
+from ..utils import in_context
+
+
+def _pandas_3_0():
+    """Silence pandas warnings and opt in to future behavior.
+
+    This sets pandas behavior to 3.0+ defaults.
+    Prior to pandas 3.0, the fillna() and replace() methods would convert
+    object columns to float64 if the resulting series was all float64.
+    In 3.0+, you need to use infer_objects() to do this.
+
+    This also opts-in to copy-on-write, which previously required `copy=False`
+    to be set.
+    """
+    if args := _pandas_3_0_options():
+        return pd.option_context(*args)
+
+    import contextlib
+
+    return contextlib.nullcontext()
+
+
+@cache
+def _pandas_3_0_options():
+    options = [
+        ("future.no_silent_downcasting", True),
+        ("mode.copy_on_write", True),
+    ]
+
+    args = []
+    for option in options:
+        try:
+            pd.get_option(option[0])
+        except KeyError:
+            continue
+        args.extend(option)
+    return args
 
 
 def _link_with_html(string, html_path=None, heading=None, pdf_format=False):
@@ -113,6 +154,7 @@ def combine_extensions(lst, html_path=None, heading_lst=None, pdf_format=True):
     return new_lst
 
 
+@in_context(_pandas_3_0())
 def drop_unused_entities(df):
     """Remove columns from a dataframe where all values in the column are NaNs.
 
@@ -130,8 +172,7 @@ def drop_unused_entities(df):
     df : pandas.DataFrame
         DataFrame with columns associated with unused entities removed.
     """
-    df = df.replace("", math.nan).dropna(axis=1, how="all").fillna("")
-    return df
+    return df.replace("", math.nan).dropna(axis=1, how="all").fillna("")
 
 
 def flatten_multiindexed_columns(df):
@@ -412,3 +453,18 @@ def num2words(integer, to="ordinal"):
         return mapper[integer]
     except KeyError:
         raise ValueError(f"Input {integer} is not supported.")
+
+
+def propagate_fence_exception(func):
+    """Decorator to prevent superfences from swallowing exceptions."""
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except Exception as e:
+            from pymdownx.superfences import SuperFencesException
+
+            raise SuperFencesException from e
+
+    return wrapper

tools/schemacode/src/bidsschematools/utils.py

@@ -5,6 +5,9 @@
 import logging
 import os
 import sys
+import warnings
+from contextlib import contextmanager
+from functools import wraps
 
 from . import data
 
@@ -149,3 +152,41 @@ def jsonschema_validator(
     validator_kwargs: ValidatorKwargs
     validator_kwargs = {"format_checker": validator_cls.FORMAT_CHECKER} if check_format else {}
     return validator_cls(schema, **validator_kwargs)  # type: ignore[call-arg]
+
+
+def in_context(context_manager):
+    """Convert a context manager into a function decorator.
+
+    Parameters
+    ----------
+    context_manager : context manager
+        The context manager to use.
+
+    Returns
+    -------
+    Callable
+        The function decorator.
+    """
+
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            with context_manager:
+                return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
+@contextmanager
+def filter_warnings(*filters):
+    """Context manager to apply warning filters.
+
+    Arguments are lists of positional arguments to :func:`warnings.filterwarnings`.
+    """
+
+    with warnings.catch_warnings():
+        for filt in filters:
+            warnings.filterwarnings(*filt)
+        yield

tools/schemacode/src/bidsschematools/validator.py

@@ -495,7 +495,7 @@ def write_report(
     """
 
     report_path = report_path.format(
-        datetime=datetime.datetime.utcnow().strftime(datetime_format),
+        datetime=datetime.datetime.now(datetime.timezone.utc).strftime(datetime_format),
         pid=os.getpid(),
     )
     report_path = os.path.abspath(os.path.expanduser(report_path))