add iwslt14_de_en data

Author: dongxqm

iwslt14_de_en/__init__.py

@@ -0,0 +1,2 @@
+from .load import load_dataset
+__all__ = ['load_dataset']

iwslt14_de_en/load.py

@@ -0,0 +1,24 @@
+import datasets
+import os
+
+
+def load_dataset(**kargs):
+    '''
+    加载缓存的数据
+    format:
+    '''
+    _CACHE_DATA_PATH = '/cognitive_comp/common_data/translation/iwslt14.de-en/'
+
+    train = datasets.load_dataset('json',
+                                  data_files=os.path.join(_CACHE_DATA_PATH, "train.de_en"),
+                                  split=datasets.Split.TRAIN)
+    val = datasets.load_dataset('json',
+                                data_files={
+                                    datasets.Split.VALIDATION: os.path.join(_CACHE_DATA_PATH, "valid.de_en")},
+                                split=datasets.Split.VALIDATION)
+    test = datasets.load_dataset('json',
+                                 data_files={
+                                     datasets.Split.TEST: os.path.join(_CACHE_DATA_PATH, "test.de_en")},
+                                 split=datasets.Split.TEST)
+
+    return datasets.DatasetDict(train=train, val=val, test=test)