We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 807592a commit 9e42551Copy full SHA for 9e42551
core/step3_2_splitbymeaning.py
@@ -79,7 +79,7 @@ def parallel_split_sentences(sentences, max_length, max_workers, nlp, retry_atte
79
for index, sentence in enumerate(sentences):
80
# Use tokenizer to split the sentence
81
tokens = tokenize_sentence(sentence, nlp)
82
- print("Tokenization result:", tokens)
+ # print("Tokenization result:", tokens)
83
num_parts = math.ceil(len(tokens) / max_length)
84
if len(tokens) > max_length:
85
future = executor.submit(split_sentence, sentence, num_parts, max_length, index=index, retry_attempt=retry_attempt)
0 commit comments