From cbe7611ca2956ffc01c98a5650b8d3a1a0bbbde6 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 22 Jan 2024 22:06:37 +0000
Subject: [PATCH] chore(deps): bump tokenizers from 0.14.1 to 0.15.1

Bumps [tokenizers](https://github.com/huggingface/tokenizers) from 0.14.1 to 0.15.1.
- [Release notes](https://github.com/huggingface/tokenizers/releases)
- [Changelog](https://github.com/huggingface/tokenizers/blob/main/RELEASE.md)
- [Commits](https://github.com/huggingface/tokenizers/compare/v0.14.1...v0.15.1)

---
updated-dependencies:
- dependency-name: tokenizers
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 Cargo.lock | 35 +++++++++++++++++++++++++++++++++--
 Cargo.toml |  2 +-
 2 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 1a80e82..bd16aea 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -980,7 +980,7 @@ dependencies = [
  "rayon",
  "serde_json",
  "tar",
- "tokenizers",
+ "tokenizers 0.14.1",
 ]
 
 [[package]]
@@ -1702,7 +1702,7 @@ dependencies = [
  "serde",
  "serde_json",
  "text-splitter",
- "tokenizers",
+ "tokenizers 0.15.1",
  "tokio",
  "tracing-actix-web",
  "zip",
@@ -2530,6 +2530,37 @@ name = "tokenizers"
 version = "0.14.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d9be88c795d8b9f9c4002b3a8f26a6d0876103a6f523b32ea3bac52d8560c17c"
+dependencies = [
+ "aho-corasick",
+ "derive_builder",
+ "esaxx-rs",
+ "getrandom",
+ "itertools 0.11.0",
+ "lazy_static",
+ "log",
+ "macro_rules_attribute",
+ "monostate",
+ "onig",
+ "paste",
+ "rand",
+ "rayon",
+ "rayon-cond",
+ "regex",
+ "regex-syntax 0.7.5",
+ "serde",
+ "serde_json",
+ "spm_precompiled",
+ "thiserror",
+ "unicode-normalization-alignments",
+ "unicode-segmentation",
+ "unicode_categories",
+]
+
+[[package]]
+name = "tokenizers"
+version = "0.15.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6db445cceba5dfeb0f9702be7d6bfd91801ddcbe8fe8722defe7f2e96da75812"
 dependencies = [
  "aho-corasick",
  "clap",
diff --git a/Cargo.toml b/Cargo.toml
index 2bfc94c..6b70ff9 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -17,7 +17,7 @@ qdrant-client = "1"
 rayon = "1"
 reqwest = { version = "0.11", default-features = false, features = ["rustls-tls", "json"] }
 serde = "1"
-tokenizers = "0.14"
+tokenizers = "0.15"
 openai-api-rs = "2.0"
 zip = "0.6"
 rust-fuzzy-search = "0.1"