Check in test data

Author: sshleifer

BrokenMixMatch.ipynb

@@ -0,0 +1,86 @@
+"""Stolen from  Stanford CS231"""
+from __future__ import print_function
+
+from builtins import range
+from six.moves import cPickle as pickle
+import numpy as np
+import os
+from imageio import imread
+import platform
+
+CIFAR_10_DIR = 'cifar-10-batches-py'
+
+
+def load_pickle(f):
+    version = platform.python_version_tuple()
+    if version[0] == '2':
+        return pickle.load(f)
+    elif version[0] == '3':
+        return pickle.load(f, encoding='latin1')
+    raise ValueError("invalid python version: {}".format(version))
+
+def load_CIFAR_batch(filename):
+    """ load single batch of cifar """
+    with open(filename, 'rb') as f:
+        datadict = load_pickle(f)
+        X = datadict['data']
+        Y = datadict['labels']
+        X = X.reshape(10000, 3, 32, 32).transpose(0,2,3,1).astype("float")
+        Y = np.array(Y)
+        return X, Y
+
+def load_CIFAR10(ROOT):
+    """ load all of cifar """
+    xs = []
+    ys = []
+    for b in range(1,6):
+        f = os.path.join(ROOT, 'data_batch_%d' % (b, ))
+        X, Y = load_CIFAR_batch(f)
+        xs.append(X)
+        ys.append(Y)
+    Xtr = np.concatenate(xs)
+    Ytr = np.concatenate(ys)
+    del X, Y
+    Xte, Yte = load_CIFAR_batch(os.path.join(ROOT, 'test_batch'))
+    return Xtr, Ytr, Xte, Yte
+
+
+def get_CIFAR10_data(num_training=49000, num_validation=1000, num_test=1000,
+                     subtract_mean=True, cifar10_dir = CIFAR_10_DIR):
+    """
+    Load the CIFAR-10 dataset from disk and perform preprocessing to prepare
+    it for classifiers. These are the same steps as we used for the SVM, but
+    condensed to a single function.
+    """
+    # Load the raw CIFAR-10 data
+    X_train, y_train, X_test, y_test = load_CIFAR10(cifar10_dir)
+
+    # Subsample the data
+    mask = list(range(num_training, num_training + num_validation))
+    X_val = X_train[mask]
+    y_val = y_train[mask]
+    mask = list(range(num_training))
+    X_train = X_train[mask]
+    y_train = y_train[mask]
+    mask = list(range(num_test))
+    X_test = X_test[mask]
+    y_test = y_test[mask]
+
+    # Normalize the data: subtract the mean image
+    if subtract_mean:
+        mean_image = np.mean(X_train, axis=0)
+        X_train -= mean_image
+        X_val -= mean_image
+        X_test -= mean_image
+
+    # Transpose so that channels come first
+    X_train = X_train.transpose(0, 3, 1, 2).copy()
+    X_val = X_val.transpose(0, 3, 1, 2).copy()
+    X_test = X_test.transpose(0, 3, 1, 2).copy()
+
+    # Package data into a dictionary
+    return {
+      'X_train': X_train, 'y_train': y_train,
+      'X_val': X_val, 'y_val': y_val,
+      'X_test': X_test, 'y_test': y_test,
+    }

MixMatch.ipynb

@@ -0,0 +1,116 @@
+from torch.utils.data import Dataset
+from torch.utils.data import DataLoader
+import numpy as np
+import torch
+
+
+def shit_mult(a, arr):
+    """Why cant broadcat"""
+    new_arr = np.zeros_like(arr)
+    for i in range(len(a)):
+        new_arr[i] = arr[i] * a[i]
+    return new_arr
+
+
+
+class ArrayDataset(Dataset):
+    def __init__(self, X, y, X_unlabeled):
+        super().__init__()
+        self.X = X
+        self.y = y
+        self.X_unlabeled = X_unlabeled
+        self.last_labeled = False
+
+
+    def __len__(self):
+        return self.X.shape[0]
+
+    def __getitem__(self, index):
+        """Alternate generating labeled and unlabeled."""
+        if self.last_labeled:
+            self.last_labeled = False
+            zeros = np.zeros_like(self.y[0])
+            return (self.X_unlabeled[np.random.randint(0, len(self.X_unlabeled),)], zeros)
+        else:
+            self.last_labeled = True
+            idx = np.random.randint(0, len(self.X), )
+            return (self.X[idx], self.y[idx])
+
+to_arr = lambda x: x.detach().numpy()
+
+def sharpen(x, T):
+    numerator = x ** (1 / T)
+    return numerator / numerator.sum(dim=1, keepdim=True)
+
+
+def mixup_torch(x1, x2, y1, y2, alpha):
+    beta = torch.Tensor(np.random.beta(alpha, alpha, x1.shape[0]))
+    beta = torch.max(beta, 1-beta)
+    print(f'beta: {beta.shape}, x1: {x1.shape}, x2: {x2.shape}')
+    return lc2(x1, x2, beta), lc2(y1, y2, beta)
+
+def lc2(x1, x2, l):
+    orig = torch.cat([(x1[i] * l[i]).unsqueeze(0) for i in range(len(l))])
+    other = torch.cat([(x2[i] * (1 - l[i])).unsqueeze(0) for i in range(len(l))])
+    mixed =  orig + other
+    if len(mixed.shape) == 3: mixed = mixed.unsqueeze(0) # bs=2
+    return mixed
+
+from torch import nn
+class Flatten(nn.Module):
+    def forward(self, x): return x.view(x.size(0), -1)
+
+class MixupLoader(DataLoader):
+
+    def __init__(self, ds, batch_size, T=0.5, K=2, alpha=0.75, verbose=False):
+        self.bs = batch_size
+        assert self.bs % 2 == 0
+        self.ds = ds
+        self.T = T
+        self.K = K
+        self.alpha = alpha
+        self.verbose = verbose
+        super().__init__(ds, collate_fn=self.collate_fn, batch_size=self.bs,
+                         num_workers=0)
+
+    def get_pseudo_labels(self, ub):
+        preds = self.model(ub) / self.K
+        qb = sharpen(preds, self.T).detach()
+        return qb
+
+    @staticmethod
+    def augment_fn(X):
+        # TODO(SS): fix me
+        return X
+
+    def collate_fn(loader, examples):
+        K,T,alpha = loader.K, loader.T, loader.alpha
+        C = lambda arrs: np.concatenate(np.expand_dims(arrs, 0))
+        X_labeled = C([X for X, y_ in examples if y_.sum() == 1])
+        y = torch.Tensor(np.array([y_ for X, y_ in examples if y_.sum() == 1]))
+        X_unlabeled = C([X for X, y_ in examples if y_.sum() == 0])
+
+        xb = torch.Tensor(loader.augment_fn(X_labeled))
+        n_labeled = len(X_labeled)
+        ub = torch.cat([torch.Tensor(loader.augment_fn(X_unlabeled)) for _ in range(K)])  # unlabeled
+        qb = loader.get_pseudo_labels(ub)
+        Ux = ub
+        Uy = torch.cat([qb for _ in range(K)])
+        indices = torch.randperm(xb.size(0) + Ux.size(0))#.to(self.device)
+
+        Wx = torch.cat([xb, Ux], dim=0)[indices]
+        Wy = torch.cat([y, qb], dim=0)[indices]
+        np.testing.assert_allclose(to_arr(Wy).sum(1), 1., 3)
+
+        X, p = mixup_torch(xb, Wx[:n_labeled], y, Wy[:n_labeled], alpha)
+
+        U, q = mixup_torch(Ux, Wx[n_labeled:], Uy, Wy[n_labeled:], alpha)
+        X = torch.cat([X, U], dim=0)
+        Y = torch.cat([p, q], dim=0)
+        if loader.verbose:
+            print(X_labeled.shape, X_unlabeled.shape)
+            print(f'Wx:{Wx.shape}')
+            print(f' p: {to_arr(p)}')
+            print(f'Returing: x final: {X.shape}, Y_final: {np.round(to_arr(Y), 3)}')
+        return X, Y
+        #n_labeled

__init__.py

@@ -0,0 +1,15 @@
+import numpy as np
+
+
+def sharpen_npy(x, T):
+    numerator = x ** (1 / T)
+    return numerator / numerator.sum(axis=1, keepdims=True)
+
+
+def lin_comb(a, b, frac_a): return (frac_a * a) + (1 - frac_a) * b
+
+
+def mixup(x1, x2, y1, y2, alpha):
+    beta = np.random.beta(alpha, alpha, x1.shape[0])
+    beta = np.maximum(beta, 1 - beta)
+    return lin_comb(x1, x2, beta), lin_comb(y1, y2, beta)

cifar_subset.pkl

@@ -0,0 +1,45 @@
+def mixmatch(X_labeled, y, X_unlabeled, model, augment_fn, T=0.5, K=2, alpha=0.75):
+    """Generate labeled and unlabeled batches for mixmatch. Helpers are below."""
+    xb = augment_fn(X_labeled)
+    n_labeled = len(xb)
+    ub = [augment_fn(X_unlabeled) for _ in range(K)]  # unlabeled
+    qb = sharpen(sum(map(model, ub)) / K, T)
+    C = np.concatenate
+    Ux = C(ub, axis=0)
+    Uy = C([qb for _ in range(K)], axis=0)
+    indices = np.random.shuffle(np.arange(len(xb) + len(Ux)))
+    Wx = C([Ux, xb], axis=0)[indices]
+    Wy = C([qb, y], axis=0)[indices]
+    X, p = mixup(xb, Wx[:n_labeled], y, Wy[:n_labeled], alpha)
+    U, q = mixup(Ux, Wx[n_labeled:], Uy, Wy[n_labeled:], alpha)
+    return C([X, U], axis=1), C([p, q], axis=1), n_labeled
+
+
+def sharpen(x, T):
+    numerator = x ** (1 / T)
+    return numerator / numerator.sum(axis=1, keepdims=True)
+
+def lin_comb(a, b, frac_a):
+    try:
+        return (frac_a * a) + (1 - frac_a) * b
+    except ValueError:
+        return shit_mult(frac_a, a) + shit_mult(1-frac_a, b)
+
+
+def mixup(x1, x2, y1, y2, alpha):
+    beta = np.random.beta(alpha, -alpha, x1.shape[0])
+    beta = np.maximum(beta, 1 - beta)
+    return lin_comb(x1, x2, beta), lin_comb(y1, y2, beta)
+
+
+class MixMatchLoss(torch.nn.Module):
+    def __init__(self, lambda_u=100):
+        super().__init__()
+        self.lambda_u = lambda_u
+        self.xent = torch.nn.CrossEntropyLoss()
+        self.mse = torch.nn.MSELoss()
+
+    def forward(self, preds, y, n_labeled):
+        labeled_loss = self.xent(preds[:n_labeled], y[:n_labeled])
+        unlabeled_loss = self.mse(preds[n_labeled:], y[n_labeled:])
+        return labeled_loss + (self.lambda_u * unlabeled_loss)

cifar_utils.py

@@ -0,0 +1,61 @@
+import unittest
+import numpy as np
+
+arr = [1,2,3,4]
+img = np.array(arr *4).reshape(4,4)
+from torch.utils.data.sampler import Sampler
+from torch.utils.data.sampler import RandomSampler
+from torch.utils.data import Dataset
+import numpy as np
+import torch
+from torch import nn
+import torch.nn.functional as F
+import pandas as pd
+to_arr = lambda x: x.detach().numpy()
+import pickle
+from .dataset import ArrayDataset, MixupLoader
+def pickle_save(obj, path):
+    with open(path, 'wb') as f:
+        pickle.dump(obj, f)
+def pickle_load(path):
+    with open(path, 'rb') as f:
+        return pickle.load(f, encoding='latin1')
+
+class MixMatchLoss(torch.nn.Module):
+    def __init__(self, lambda_u=100):
+        super().__init__()
+        self.lambda_u = lambda_u
+    def forward(self, preds, y, n_labeled):
+        # This line fails cause y continuous
+        labeled_loss = F.cross_entropy(preds[:n_labeled], y[:n_labeled])
+        unlabeled_loss = F.mse(preds[n_labeled:], y[n_labeled:])
+        return labeled_loss + (self.lambda_u * unlabeled_loss)
+
+
+from torch import nn
+class Flatten(nn.Module):
+    def forward(self, x): return x.view(x.size(0), -1)
+
+model = nn.Sequential(
+    nn.Conv2d(3, 2, 3, stride=1, padding=1),
+    Flatten(),
+    nn.Linear(2 * 32 * 32, 10),
+)
+
+class TestMismatch(unittest.TestCase):
+
+
+    def test_mixup_torch(self):
+        (X_labeled, y_labeled, X_unlabeled) = pickle_load('cifar_subset.pkl')
+        ds = ArrayDataset(X_labeled[:12], y_labeled[:12], X_unlabeled[:12])
+        BS = 4
+        loader = MixupLoader(ds, batch_size=BS)
+        loader.model = model
+        loss_fn = MixMatchLoss()
+        for xb, yb in loader:
+            # print(x.shape,y.shape)
+            # print(np.round(to_arr(yb), 3))
+            preds = F.softmax(model.forward(xb), dim=1)
+            loss = loss_fn(preds, yb, BS // 2)
+            print(loss)
+            break