-
Notifications
You must be signed in to change notification settings - Fork 25
/
datasets.py
65 lines (53 loc) · 2.88 KB
/
datasets.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
import os.path as osp
import numpy as np
import bcolz
from torchvision import datasets, transforms
def bcolz_save(path, np_array):
c = bcolz.carray(np_array, rootdir=path, mode='w')
c.flush()
print("Saved to " + path)
get_mnist = False
if get_mnist:
DATA_DIR = "../../data/mnist/"
TRN_INPUTS_BCOLZ_PATH = osp.join(DATA_DIR, "trn_inputs.bcolz")
TRN_TARGETS_BCOLZ_PATH = osp.join(DATA_DIR, "trn_targets.bcolz")
TST_INPUTS_BCOLZ_PATH = osp.join(DATA_DIR, "tst_inputs.bcolz")
TST_TARGETS_BCOLZ_PATH = osp.join(DATA_DIR, "tst_targets.bcolz")
transform = transforms.Compose([transforms.ToTensor(),
transforms.Normalize((0.1307,),
(0.3081,))])
trn_dataset_tensor = datasets.MNIST(DATA_DIR, train=True, download=True,
transform=transform)
tst_dataset_tensor = datasets.MNIST(DATA_DIR, train=False, download=True,
transform=transform)
trn_inputs_np = np.array([x.numpy() for x, y in list(trn_dataset_tensor)])
trn_targets_np = np.array([y for x, y in trn_dataset_tensor])
tst_inputs_np = np.array([x.numpy() for x, y in list(tst_dataset_tensor)])
tst_targets_np = np.array([y for x, y in tst_dataset_tensor])
bcolz_save(TRN_INPUTS_BCOLZ_PATH, trn_inputs_np)
bcolz_save(TRN_TARGETS_BCOLZ_PATH, trn_targets_np)
bcolz_save(TST_INPUTS_BCOLZ_PATH, tst_inputs_np)
bcolz_save(TST_TARGETS_BCOLZ_PATH, tst_targets_np)
get_cifar10 = False
if get_cifar10:
DATA_DIR = "../../data/cifar10/"
TRN_INPUTS_BCOLZ_PATH = osp.join(DATA_DIR, "trn_inputs.bcolz")
TRN_TARGETS_BCOLZ_PATH = osp.join(DATA_DIR, "trn_targets.bcolz")
TST_INPUTS_BCOLZ_PATH = osp.join(DATA_DIR, "tst_inputs.bcolz")
TST_TARGETS_BCOLZ_PATH = osp.join(DATA_DIR, "tst_targets.bcolz")
transform = transforms.Compose([transforms.ToTensor(),
transforms.Normalize(
(0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010))])
trn_dataset_tensor = datasets.CIFAR10(DATA_DIR, train=True, download=True,
transform=transform)
tst_dataset_tensor = datasets.CIFAR10(DATA_DIR, train=False, download=True,
transform=transform)
trn_inputs_np = np.array([x.numpy() for x, y in list(trn_dataset_tensor)])
trn_targets_np = np.array([y for x, y in trn_dataset_tensor])
tst_inputs_np = np.array([x.numpy() for x, y in list(tst_dataset_tensor)])
tst_targets_np = np.array([y for x, y in tst_dataset_tensor])
bcolz_save(TRN_INPUTS_BCOLZ_PATH, trn_inputs_np)
bcolz_save(TRN_TARGETS_BCOLZ_PATH, trn_targets_np)
bcolz_save(TST_INPUTS_BCOLZ_PATH, tst_inputs_np)
bcolz_save(TST_TARGETS_BCOLZ_PATH, tst_targets_np)