Skip to content

Update main_lm_no_trainer.py to load dataset from the HF Hub #195

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Update main_lm_no_trainer.py to load dataset from the HF Hub #195

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e88e27a
Update main_lm_no_trainer.py to load dataset from the HF Hub
fenchri Sep 20, 2023
7fd7744
Update main_lm_no_trainer.py
fenchri Sep 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 7 additions & 9 deletions NLP/EntityCS/cs_pretrain/main_lm_no_trainer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
import numpy as np
import datasets
import torch
from datasets import load_from_disk, concatenate_datasets
from datasets import load_dataset, concatenate_datasets
from torch.utils.data import DataLoader

import transformers
Expand Down Expand Up @@ -378,10 +378,12 @@ def main():
raw_datasets_by_lang = {}
for lang in args.languages:
logger.info(f"Loading {lang.upper()}")

raw_datasets_by_lang[lang] = load_dataset(
"huawei-noah/entity_cs", data_dir=f"data/{lang}/", split="train"
)

if lang == "en":
raw_datasets_by_lang[lang] = load_from_disk(
os.path.join(args.hf_datasets_folder_en, "en")
)
# EN dataset does not have "language" column, so we add it now
new_col = [lang] * len(raw_datasets_by_lang[lang])
raw_datasets_by_lang[lang] = raw_datasets_by_lang[lang].add_column(
Expand All @@ -392,16 +394,12 @@ def main():
"en_sentence", "cs_sentence"
)
else:
raw_datasets_by_lang[lang] = load_from_disk(
os.path.join(args.hf_datasets_folder_no_en, lang)
)
# We are not feeding parallel data, so remove en_sentence from the dataset
raw_datasets_by_lang[lang] = raw_datasets_by_lang[lang].remove_columns(
"en_sentence"
)

# shuffle cs_sentences in all langs
for lang in args.languages:
# shuffle cs_sentences in all langs
raw_datasets_by_lang[lang] = raw_datasets_by_lang[lang].shuffle(seed=args.seed)

# merge langs to one train, validation dataset
Expand Down