Refactor model module, add new net

Author: Enderdead

demos/BIN_CIFAR10.py

@@ -3,16 +3,16 @@
 sys.path.insert(0, os.path.abspath('..'))
 
 import torch
-from models.Alexnet_Bin import *
-from models.Alexnet_BinTest import *
+from models.Alexnet.Alexnet_Bin import *
+from models.Alexnet.Alexnet_BinTest import *
 from torchvision import transforms
 import torchvision
 
 
 from device import device
 
 
-BATCH_SIZE = 128
+BATCH_SIZE = 3
 
 model = AlexNetBin().to(device)
 

demos/BIN_MNIST.py demos/BIN_MNIST_conv.pydemos/BIN_MNIST.py renamed to demos/BIN_MNIST_conv.py

@@ -2,12 +2,13 @@
 import sys
 sys.path.insert(0, os.path.abspath('..'))
 
-
-from models.binMNIST import BinMNIST
+#.9833
+from models.ConvNet.binMNIST_conv import BinMNIST
 import torch
 import torch.utils.data
 import numpy as np
 import torchvision
+from progress.bar import ChargingBar
 
 
 

demos/BIN_MNIST.1.py demos/BIN_MNIST_dense.pydemos/BIN_MNIST.1.py renamed to demos/BIN_MNIST_dense.py

@@ -2,8 +2,8 @@
 import sys
 sys.path.insert(0, os.path.abspath('..'))
 
-
-from models.binMNIST_conv import BinMNIST
+# 0.9623
+from models.FullNet.binMNIST import BinMNIST
 import torch
 import torch.utils.data
 import numpy as np

demos/LIN_CIFAR10.py

@@ -3,7 +3,7 @@
 sys.path.insert(0, os.path.abspath('..'))
 # 82
 import torch
-from models.VGG_LinQuant import VGGLinQuant
+from models.VGG.VGG_LinQuant import VGGLinQuant
 from torchvision import transforms
 import torchvision
 from progress.bar import ChargingBar

demos/LIN_MNIST.py

@@ -3,13 +3,12 @@
 sys.path.insert(0, os.path.abspath('..'))
 
 
-from models.linQuantMNIST import LinQuantMNIST
+from models.FullNet.linQuantMNIST import LinQuantMNIST
 import torch
 import torch.utils.data
 import numpy as np
 import torchvision
 
-
 from device import device
 
 
@@ -18,6 +17,8 @@
 MOMENTUM = 0.6
 EPOCH = 600
 DATASET_SIZE = 60000
+
+
 def adjust_learning_rate(optimizer, epoch):
     global LEARNING_RATE
     """Sets the learning rate to the initial LR decayed by 10 every 30 epochs"""

models/Alexnet.py models/Alexnet/Alexnet.pymodels/Alexnet.py renamed to models/Alexnet/Alexnet.py

@@ -36,8 +36,6 @@ def __init__(self, num_classes=10):
                 nn.MaxPool2d(kernel_size=3, stride=2),
                 torch.nn.BatchNorm2d(256),
                 nn.Hardtanh(inplace=True),
-
-
         )
         self.classifieur = nn.Sequential(
                 BinaryConnect(stochastic=False),

models/Alexnet_Bin.py models/Alexnet/Alexnet_Bin.pymodels/Alexnet_Bin.py renamed to models/Alexnet/Alexnet_Bin.py

@@ -0,0 +1,196 @@
+
+
+
+
+import torch.nn as nn
+import torchvision.transforms as transforms
+import torch
+import pdb
+import torch.nn as nn
+import math
+from torch.autograd import Variable
+from torch.autograd import Function
+import numpy as np
+
+
+def Binarize(tensor,quant_mode='det'):
+    if quant_mode=='det':
+        return tensor.sign()
+    else:
+        return tensor.add_(1).div_(2).add_(torch.rand(tensor.size()).add(-0.5)).clamp_(0,1).round().mul_(2).add_(-1)
+
+
+
+
+class HingeLoss(nn.Module):
+    def __init__(self):
+        super(HingeLoss,self).__init__()
+        self.margin=1.0
+
+    def hinge_loss(self,input,target):
+            #import pdb; pdb.set_trace()
+            output=self.margin-input.mul(target)
+            output[output.le(0)]=0
+            return output.mean()
+
+    def forward(self, input, target):
+        return self.hinge_loss(input,target)
+
+class SqrtHingeLossFunction(Function):
+    def __init__(self):
+        super(SqrtHingeLossFunction,self).__init__()
+        self.margin=1.0
+
+    def forward(self, input, target):
+        output=self.margin-input.mul(target)
+        output[output.le(0)]=0
+        self.save_for_backward(input, target)
+        loss=output.mul(output).sum(0).sum(1).div(target.numel())
+        return loss
+
+    def backward(self,grad_output):
+       input, target = self.saved_tensors
+       output=self.margin-input.mul(target)
+       output[output.le(0)]=0
+       import pdb; pdb.set_trace()
+       grad_output.resize_as_(input).copy_(target).mul_(-2).mul_(output)
+       grad_output.mul_(output.ne(0).float())
+       grad_output.div_(input.numel())
+       return grad_output,grad_output
+
+def Quantize(tensor,quant_mode='det',  params=None, numBits=8):
+    tensor.clamp_(-2**(numBits-1),2**(numBits-1))
+    if quant_mode=='det':
+        tensor=tensor.mul(2**(numBits-1)).round().div(2**(numBits-1))
+    else:
+        tensor=tensor.mul(2**(numBits-1)).round().add(torch.rand(tensor.size()).add(-0.5)).div(2**(numBits-1))
+        quant_fixed(tensor, params)
+    return tensor
+
+import torch.nn._functions as tnnf
+
+
+class BinarizeLinear(nn.Linear):
+
+    def __init__(self, *kargs, **kwargs):
+        super(BinarizeLinear, self).__init__(*kargs, **kwargs)
+
+    def forward(self, input):
+
+        if input.size(1) != 784:
+            input.data=Binarize(input.data)
+        if not hasattr(self.weight,'org'):
+            self.weight.org=self.weight.data.clone()
+        self.weight.data=Binarize(self.weight.org)
+        out = nn.functional.linear(input, self.weight)
+        if not self.bias is None:
+            self.bias.org=self.bias.data.clone()
+            out += self.bias.view(1, -1).expand_as(out)
+
+        return out
+
+class BinarizeConv2d(nn.Conv2d):
+
+    def __init__(self, *kargs, **kwargs):
+        super(BinarizeConv2d, self).__init__(*kargs, **kwargs)
+
+
+    def forward(self, input):
+        if input.size(1) != 3:
+            input.data = Binarize(input.data)
+        if not hasattr(self.weight,'org'):
+            self.weight.org=self.weight.data.clone()
+        self.weight.data=Binarize(self.weight.org)
+
+        out = nn.functional.conv2d(input, self.weight, None, self.stride,
+                                   self.padding, self.dilation, self.groups)
+
+        if not self.bias is None:
+            self.bias.org=self.bias.data.clone()
+            out += self.bias.view(1, -1, 1, 1).expand_as(out)
+
+        return out
+
+
+class AlexNetOWT_BN(nn.Module):
+
+    def __init__(self, num_classes=10):
+        super(AlexNetOWT_BN, self).__init__()
+        print("lol")
+        self.ratioInfl=3
+        self.features = nn.Sequential(
+            BinarizeConv2d(3, int(64*self.ratioInfl), kernel_size=11, stride=4, padding=2),
+            nn.MaxPool2d(kernel_size=3, stride=2),
+            nn.BatchNorm2d(int(64*self.ratioInfl)),
+            nn.Hardtanh(inplace=True),
+            BinarizeConv2d(int(64*self.ratioInfl), int(192*self.ratioInfl), kernel_size=5, padding=2),
+            nn.MaxPool2d(kernel_size=3, stride=2),
+            nn.BatchNorm2d(int(192*self.ratioInfl)),
+            nn.Hardtanh(inplace=True),
+
+            BinarizeConv2d(int(192*self.ratioInfl), int(384*self.ratioInfl), kernel_size=3, padding=1),
+            nn.BatchNorm2d(int(384*self.ratioInfl)),
+            nn.Hardtanh(inplace=True),
+
+            BinarizeConv2d(int(384*self.ratioInfl), int(256*self.ratioInfl), kernel_size=3, padding=1),
+            nn.BatchNorm2d(int(256*self.ratioInfl)),
+            nn.Hardtanh(inplace=True),
+
+            BinarizeConv2d(int(256*self.ratioInfl), 256, kernel_size=3, padding=1),
+            nn.MaxPool2d(kernel_size=3, stride=2),
+            nn.BatchNorm2d(256),
+            nn.Hardtanh(inplace=True)
+
+        )
+        self.classifier = nn.Sequential(
+            BinarizeLinear(256 * 6 * 6, 4096),
+            nn.BatchNorm1d(4096),
+            nn.Hardtanh(inplace=True),
+            #nn.Dropout(0.5),
+            BinarizeLinear(4096, 4096),
+            nn.BatchNorm1d(4096),
+            nn.Hardtanh(inplace=True),
+            #nn.Dropout(0.5),
+            BinarizeLinear(4096, 10),
+            nn.BatchNorm1d(10),
+            nn.LogSoftmax()
+        )
+
+        #self.regime = {
+        #    0: {'optimizer': 'SGD', 'lr': 1e-2,
+        #        'weight_decay': 5e-4, 'momentum': 0.9},
+        #    10: {'lr': 5e-3},
+        #    15: {'lr': 1e-3, 'weight_decay': 0},
+        #    20: {'lr': 5e-4},
+        #    25: {'lr': 1e-4}
+        #}
+        self.regime = {
+            0: {'optimizer': 'Adam', 'lr': 5e-3},
+            20: {'lr': 1e-3},
+            30: {'lr': 5e-4},
+            35: {'lr': 1e-4},
+            40: {'lr': 1e-5}
+        }
+        normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
+                                         std=[0.229, 0.224, 0.225])
+        self.input_transform = {
+            'train': transforms.Compose([
+                transforms.Scale(256),
+                transforms.RandomCrop(224),
+                transforms.RandomHorizontalFlip(),
+                transforms.ToTensor(),
+                normalize
+            ]),
+            'eval': transforms.Compose([
+                transforms.Scale(256),
+                transforms.CenterCrop(224),
+                transforms.ToTensor(),
+                normalize
+            ])
+        }
+
+    def forward(self, x):
+        x = self.features(x)
+        x = x.view(-1, 256 * 6 * 6)
+        x = self.classifier(x)
+        return x

models/Alexnet/Alexnet_BinTest.py

@@ -32,8 +32,8 @@ def reset(self):
 
 
     def clamp(self):
-        #self.linear1.clamp()
-        #self.linear2.clamp()
+        self.conv1.clamp()
+        self.conv2.clamp()
         self.linear3.clamp()
         self.linear4.clamp()