From e3b2733238e280ed0fc81c40eb2d8eacf36213c2 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 15 Jan 2024 22:42:14 +0000
Subject: [PATCH] chore(deps): bump tokenizers from 0.14.1 to 0.15.0

Bumps [tokenizers](https://github.com/huggingface/tokenizers) from 0.14.1 to 0.15.0.
- [Release notes](https://github.com/huggingface/tokenizers/releases)
- [Changelog](https://github.com/huggingface/tokenizers/blob/main/RELEASE.md)
- [Commits](https://github.com/huggingface/tokenizers/compare/v0.14.1...v0.15.0)

---
updated-dependencies:
- dependency-name: tokenizers
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 Cargo.lock | 35 +++++++++++++++++++++++++++++++++--
 Cargo.toml |  2 +-
 2 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 42cc74b..8dd9833 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -987,7 +987,7 @@ dependencies = [
  "rayon",
  "serde_json",
  "tar",
- "tokenizers",
+ "tokenizers 0.14.1",
 ]
 
 [[package]]
@@ -1709,7 +1709,7 @@ dependencies = [
  "serde",
  "serde_json",
  "text-splitter",
- "tokenizers",
+ "tokenizers 0.15.0",
  "tokio",
  "tracing-actix-web",
  "zip",
@@ -2548,6 +2548,37 @@ name = "tokenizers"
 version = "0.14.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d9be88c795d8b9f9c4002b3a8f26a6d0876103a6f523b32ea3bac52d8560c17c"
+dependencies = [
+ "aho-corasick",
+ "derive_builder",
+ "esaxx-rs",
+ "getrandom",
+ "itertools 0.11.0",
+ "lazy_static",
+ "log",
+ "macro_rules_attribute",
+ "monostate",
+ "onig",
+ "paste",
+ "rand",
+ "rayon",
+ "rayon-cond",
+ "regex",
+ "regex-syntax 0.7.5",
+ "serde",
+ "serde_json",
+ "spm_precompiled",
+ "thiserror",
+ "unicode-normalization-alignments",
+ "unicode-segmentation",
+ "unicode_categories",
+]
+
+[[package]]
+name = "tokenizers"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "062b8a9613d6017633b80fb55fbb33f1aff006c36225a3025630753398034b3c"
 dependencies = [
  "aho-corasick",
  "clap",
diff --git a/Cargo.toml b/Cargo.toml
index 2bfc94c..6b70ff9 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -17,7 +17,7 @@ qdrant-client = "1"
 rayon = "1"
 reqwest = { version = "0.11", default-features = false, features = ["rustls-tls", "json"] }
 serde = "1"
-tokenizers = "0.14"
+tokenizers = "0.15"
 openai-api-rs = "2.0"
 zip = "0.6"
 rust-fuzzy-search = "0.1"