Update develop from master for v3.7

requirements.txt

@@ -18,7 +18,7 @@ numpy>=1.15.0; python_version < "3.9"
 numpy>=1.19.0; python_version >= "3.9"
 requests>=2.13.0,<3.0.0
 tqdm>=4.38.0,<5.0.0
-pydantic>=1.7.4,!=1.8,!=1.8.1,<1.11.0
+pydantic>=1.7.4,!=1.8,!=1.8.1,<3.0.0
 jinja2
 langcodes>=3.2.0,<4.0.0
 # Official Python utilities

setup.cfg

@@ -65,7 +65,7 @@ install_requires =
     numpy>=1.15.0; python_version < "3.9"
     numpy>=1.19.0; python_version >= "3.9"
     requests>=2.13.0,<3.0.0
-    pydantic>=1.7.4,!=1.8,!=1.8.1,<1.11.0
+    pydantic>=1.7.4,!=1.8,!=1.8.1,<3.0.0
     jinja2
     # Official Python utilities
     setuptools
@@ -116,6 +116,8 @@ cuda117 =
     cupy-cuda117>=5.0.0b4,<13.0.0
 cuda11x =
     cupy-cuda11x>=11.0.0,<13.0.0
+cuda12x =
+    cupy-cuda12x>=11.5.0,<13.0.0
 cuda-autodetect =
     cupy-wheel>=11.0.0,<13.0.0
 apple =

setup.py

@@ -1,10 +1,9 @@
 #!/usr/bin/env python
 from setuptools import Extension, setup, find_packages
 import sys
-import platform
 import numpy
-from distutils.command.build_ext import build_ext
-from distutils.sysconfig import get_python_inc
+from setuptools.command.build_ext import build_ext
+from sysconfig import get_path
 from pathlib import Path
 import shutil
 from Cython.Build import cythonize
@@ -88,30 +87,6 @@
 }
 
 
-def is_new_osx():
-    """Check whether we're on OSX >= 10.7"""
-    if sys.platform != "darwin":
-        return False
-    mac_ver = platform.mac_ver()[0]
-    if mac_ver.startswith("10"):
-        minor_version = int(mac_ver.split(".")[1])
-        if minor_version >= 7:
-            return True
-        else:
-            return False
-    return False
-
-
-if is_new_osx():
-    # On Mac, use libc++ because Apple deprecated use of
-    # libstdc
-    COMPILE_OPTIONS["other"].append("-stdlib=libc++")
-    LINK_OPTIONS["other"].append("-lc++")
-    # g++ (used by unix compiler on mac) links to libstdc++ as a default lib.
-    # See: https://stackoverflow.com/questions/1653047/avoid-linking-to-libstdc
-    LINK_OPTIONS["other"].append("-nodefaultlibs")
-
-
 # By subclassing build_extensions we have the actual compiler that will be used which is really known only after finalize_options
 # http://stackoverflow.com/questions/724664/python-distutils-how-to-get-a-compiler-that-is-going-to-be-used
 class build_ext_options:
@@ -204,7 +179,7 @@ def setup_package():
 
     include_dirs = [
         numpy.get_include(),
-        get_python_inc(plat_specific=True),
+        get_path("include"),
     ]
     ext_modules = []
     ext_modules.append(

spacy/cli/__init__.py

@@ -14,6 +14,7 @@
 from .debug_model import debug_model  # noqa: F401
 from .download import download  # noqa: F401
 from .evaluate import evaluate  # noqa: F401
+from .find_function import find_function  # noqa: F401
 from .find_threshold import find_threshold  # noqa: F401
 from .info import info  # noqa: F401
 from .init_config import fill_config, init_config  # noqa: F401

spacy/cli/assemble.py

@@ -40,7 +40,8 @@ def assemble_cli(
 
     DOCS: https://spacy.io/api/cli#assemble
     """
-    util.logger.setLevel(logging.DEBUG if verbose else logging.INFO)
+    if verbose:
+        util.logger.setLevel(logging.DEBUG)
     # Make sure all files and paths exists if they are needed
     if not config_path or (str(config_path) != "-" and not config_path.exists()):
         msg.fail("Config file not found", config_path, exits=1)

spacy/cli/evaluate.py

@@ -28,6 +28,7 @@ def evaluate_cli(
     displacy_path: Optional[Path] = Opt(None, "--displacy-path", "-dp", help="Directory to output rendered parses as HTML", exists=True, file_okay=False),
     displacy_limit: int = Opt(25, "--displacy-limit", "-dl", help="Limit of parses to render as HTML"),
     per_component: bool = Opt(False, "--per-component", "-P", help="Return scores per component, only applicable when an output JSON file is specified."),
+    spans_key: str = Opt("sc", "--spans-key", "-sk", help="Spans key to use when evaluating Doc.spans"),
     # fmt: on
 ):
     """
@@ -53,6 +54,7 @@ def evaluate_cli(
         displacy_limit=displacy_limit,
         per_component=per_component,
         silent=False,
+        spans_key=spans_key,
     )
 
 

spacy/cli/find_function.py

@@ -0,0 +1,69 @@
+from typing import Optional, Tuple
+
+from catalogue import RegistryError
+from wasabi import msg
+
+from ..util import registry
+from ._util import Arg, Opt, app
+
+
+@app.command("find-function")
+def find_function_cli(
+    # fmt: off
+    func_name: str = Arg(..., help="Name of the registered function."),
+    registry_name: Optional[str] = Opt(None, "--registry", "-r", help="Name of the catalogue registry."),
+    # fmt: on
+):
+    """
+    Find the module, path and line number to the file the registered
+    function is defined in, if available.
+
+    func_name (str): Name of the registered function.
+    registry_name (Optional[str]): Name of the catalogue registry.
+
+    DOCS: https://spacy.io/api/cli#find-function
+    """
+    if not registry_name:
+        registry_names = registry.get_registry_names()
+        for name in registry_names:
+            if registry.has(name, func_name):
+                registry_name = name
+                break
+
+    if not registry_name:
+        msg.fail(
+            f"Couldn't find registered function: '{func_name}'",
+            exits=1,
+        )
+
+    assert registry_name is not None
+    find_function(func_name, registry_name)
+
+
+def find_function(func_name: str, registry_name: str) -> Tuple[str, int]:
+    registry_desc = None
+    try:
+        registry_desc = registry.find(registry_name, func_name)
+    except RegistryError as e:
+        msg.fail(
+            f"Couldn't find registered function: '{func_name}' in registry '{registry_name}'",
+        )
+        msg.fail(f"{e}", exits=1)
+    assert registry_desc is not None
+
+    registry_path = None
+    line_no = None
+    if registry_desc["file"]:
+        registry_path = registry_desc["file"]
+        line_no = registry_desc["line_no"]
+
+    if not registry_path or not line_no:
+        msg.fail(
+            f"Couldn't find path to registered function: '{func_name}' in registry '{registry_name}'",
+            exits=1,
+        )
+    assert registry_path is not None
+    assert line_no is not None
+
+    msg.good(f"Found registered function '{func_name}' at {registry_path}:{line_no}")
+    return str(registry_path), int(line_no)

spacy/cli/find_threshold.py

@@ -52,8 +52,8 @@ def find_threshold_cli(
 
     DOCS: https://spacy.io/api/cli#find-threshold
     """
-
-    util.logger.setLevel(logging.DEBUG if verbose else logging.INFO)
+    if verbose:
+        util.logger.setLevel(logging.DEBUG)
     import_code(code_path)
     find_threshold(
         model=model,

spacy/cli/init_pipeline.py

@@ -39,7 +39,8 @@ def init_vectors_cli(
     you can use in the [initialize] block of your config to initialize
     a model with vectors.
     """
-    util.logger.setLevel(logging.DEBUG if verbose else logging.INFO)
+    if verbose:
+        util.logger.setLevel(logging.DEBUG)
     msg.info(f"Creating blank nlp object for language '{lang}'")
     nlp = util.get_lang_class(lang)()
     if jsonl_loc is not None:
@@ -87,7 +88,8 @@ def init_pipeline_cli(
     use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU")
     # fmt: on
 ):
-    util.logger.setLevel(logging.DEBUG if verbose else logging.INFO)
+    if verbose:
+        util.logger.setLevel(logging.DEBUG)
     overrides = parse_config_overrides(ctx.args)
     import_code(code_path)
     setup_gpu(use_gpu)
@@ -116,7 +118,8 @@ def init_labels_cli(
     """Generate JSON files for the labels in the data. This helps speed up the
     training process, since spaCy won't have to preprocess the data to
     extract the labels."""
-    util.logger.setLevel(logging.DEBUG if verbose else logging.INFO)
+    if verbose:
+        util.logger.setLevel(logging.DEBUG)
     if not output_path.exists():
         output_path.mkdir(parents=True)
     overrides = parse_config_overrides(ctx.args)

spacy/cli/package.py

@@ -403,7 +403,7 @@ def _format_sources(data: Any) -> str:
         if author:
             result += " ({})".format(author)
         sources.append(result)
-    return "<br />".join(sources)
+    return "<br>".join(sources)
 
 
 def _format_accuracy(data: Dict[str, Any], exclude: List[str] = ["speed"]) -> str:

spacy/cli/train.py

@@ -47,7 +47,8 @@ def train_cli(
 
     DOCS: https://spacy.io/api/cli#train
     """
-    util.logger.setLevel(logging.DEBUG if verbose else logging.INFO)
+    if verbose:
+        util.logger.setLevel(logging.DEBUG)
     overrides = parse_config_overrides(ctx.args)
     import_code(code_path)
     train(config_path, output_path, use_gpu=use_gpu, overrides=overrides)

spacy/displacy/render.py

@@ -313,6 +313,8 @@ def render(
                 self.lang = settings.get("lang", DEFAULT_LANG)
             render_id = f"{id_prefix}-{i}"
             svg = self.render_svg(render_id, p["words"], p["arcs"])
+            if p.get("title"):
+                svg = TPL_TITLE.format(title=p.get("title")) + svg
             rendered.append(svg)
         if page:
             content = "".join([TPL_FIGURE.format(content=svg) for svg in rendered])
@@ -565,7 +567,7 @@ def render_ents(
             for i, fragment in enumerate(fragments):
                 markup += escape_html(fragment)
                 if len(fragments) > 1 and i != len(fragments) - 1:
-                    markup += "</br>"
+                    markup += "<br>"
             if self.ents is None or label.upper() in self.ents:
                 color = self.colors.get(label.upper(), self.default_color)
                 ent_settings = {
@@ -583,7 +585,7 @@ def render_ents(
         for i, fragment in enumerate(fragments):
             markup += escape_html(fragment)
             if len(fragments) > 1 and i != len(fragments) - 1:
-                markup += "</br>"
+                markup += "<br>"
         markup = TPL_ENTS.format(content=markup, dir=self.direction)
         if title:
             markup = TPL_TITLE.format(title=title) + markup

spacy/lang/tr/examples.py

@@ -15,4 +15,7 @@
     "Türkiye'nin başkenti neresi?",
     "Bakanlar Kurulu 180 günlük eylem planını açıkladı.",
     "Merkez Bankası, beklentiler doğrultusunda faizlerde değişikliğe gitmedi.",
+    "Cemal Sureya kimdir?",
+    "Bunlari Biliyor muydunuz?",
+    "Altinoluk Turkiye haritasinin neresinde yer alir?",
 ]

spacy/ml/models/tok2vec.py

@@ -67,8 +67,8 @@ def build_hash_embed_cnn_tok2vec(
         are between 2 and 8.
     window_size (int): The number of tokens on either side to concatenate during
         the convolutions. The receptive field of the CNN will be
-        depth * (window_size * 2 + 1), so a 4-layer network with window_size of
-        2 will be sensitive to 20 words at a time. Recommended value is 1.
+        depth * window_size * 2 + 1, so a 4-layer network with window_size of
+        2 will be sensitive to 17 words at a time. Recommended value is 1.
     embed_size (int): The number of rows in the hash embedding tables. This can
         be surprisingly small, due to the use of the hash embeddings. Recommended
         values are between 2000 and 10000.

spacy/pipeline/_edit_tree_internals/schemas.py

@@ -1,8 +1,12 @@
 from collections import defaultdict
 from typing import Any, Dict, List, Union
 
-from pydantic import BaseModel, Field, ValidationError
-from pydantic.types import StrictBool, StrictInt, StrictStr
+try:
+    from pydantic.v1 import BaseModel, Field, ValidationError
+    from pydantic.v1.types import StrictBool, StrictInt, StrictStr
+except ImportError:
+    from pydantic import BaseModel, Field, ValidationError  # type: ignore
+    from pydantic.types import StrictBool, StrictInt, StrictStr  # type: ignore
 
 
 class MatchNodeSchema(BaseModel):

spacy/schemas.py

@@ -16,19 +16,34 @@
     Union,
 )
 
-from pydantic import (
-    BaseModel,
-    ConstrainedStr,
-    Field,
-    StrictBool,
-    StrictFloat,
-    StrictInt,
-    StrictStr,
-    ValidationError,
-    create_model,
-    validator,
-)
-from pydantic.main import ModelMetaclass
+try:
+    from pydantic.v1 import (
+        BaseModel,
+        ConstrainedStr,
+        Field,
+        StrictBool,
+        StrictFloat,
+        StrictInt,
+        StrictStr,
+        ValidationError,
+        create_model,
+        validator,
+    )
+    from pydantic.v1.main import ModelMetaclass
+except ImportError:
+    from pydantic import (  # type: ignore
+        BaseModel,
+        ConstrainedStr,
+        Field,
+        StrictBool,
+        StrictFloat,
+        StrictInt,
+        StrictStr,
+        ValidationError,
+        create_model,
+        validator,
+    )
+    from pydantic.main import ModelMetaclass  # type: ignore
 from thinc.api import ConfigValidationError, Model, Optimizer
 from thinc.config import Promise
 

spacy/tests/pipeline/test_initialize.py

@@ -1,5 +1,10 @@
 import pytest
-from pydantic import StrictBool
+
+try:
+    from pydantic.v1 import StrictBool
+except ImportError:
+    from pydantic import StrictBool  # type: ignore
+
 from thinc.api import ConfigValidationError
 
 from spacy.lang.en import English

spacy/tests/pipeline/test_pipe_factories.py

@@ -1,5 +1,10 @@
 import pytest
-from pydantic import StrictInt, StrictStr
+
+try:
+    from pydantic.v1 import StrictInt, StrictStr
+except ImportError:
+    from pydantic import StrictInt, StrictStr  # type: ignore
+
 from thinc.api import ConfigValidationError, Linear, Model
 
 import spacy