added tests for activation functions

Author: Hiteshi

MLlib/activations.py

@@ -253,7 +253,7 @@ def derivative(X, alpha=0.01):
         ndarray(dtype=float,ndim=1)
             Outputs array of derivatives.
         """
-        dx = np.greater(X,0).astype(float)
+        dx = np.greater(X, 0).astype(float)
         dx[X < 0] = -alpha
         return dx
 

MLlib/optimizers.py

@@ -5,7 +5,7 @@
 
 class GradientDescent():
     """
-    
+
     A classic gradient descent implementation.
 
     W = W - a * dm

MLlib/tests/test_activations.py

@@ -1,6 +1,6 @@
 import numpy as np
-from MLlib.activations import Sigmoid, Relu
-from MLlib.activations import unit_step,TanH,LeakyRelu,Elu
+from MLlib.activations import Sigmoid, Relu, Softsign, Swish
+from MLlib.activations import unit_step, TanH, LeakyRelu, Elu, Softmax
 
 
 def test_Sigmoid():
@@ -42,7 +42,7 @@ def test_unit_step():
 
 
 def test_TanH():
-    X= np.array([[1, -2, 3], [-1, 2, 1],[0, -5, 6]])
+    X = np.array([[1, -2, 3], [-1, 2, 1], [0, -5, 6]])
     if np.array_equal(
         np.tanh(X),
         TanH.activation(X)
@@ -56,26 +56,74 @@ def test_TanH():
 
 
 def test_LeakyRelu(alpha):
-    X= np.array([[1, -2, 3], [-1, 2, 1],[0, -5, 6]])
+    X = np.array([[1, -2, 3], [-1, 2, 1], [0, -5, 6]])
     if np.array_equal(
-        np.maximum(alpha*X,X),
-        LeakyRelu.activation(X,alpha)
+        np.maximum(alpha*X, X),
+        LeakyRelu.activation(X, alpha)
     ) is not True:
         raise AssertionError
     dx = np.greater(X, 0).astype(float)
-    dx[X<0]=-alpha
+    dx[X < 0] = -alpha
     if np.array_equal(
         dx,
-        LeakyRelu.derivative(X,alpha)
+        LeakyRelu.derivative(X, alpha)
     ) is not True:
         raise AssertionError
 
-test_LeakyRelu(0.01)
+
 def test_Elu(alpha):
-    X= np.array([[1, -2, 3], [-1, 2, 1],[0, -5, 6]])
+    X = np.array([[1, -2, 3], [-1, 2, 1], [0, -5, 6]])
+    if np.array_equal(
+        np.maximum(X, 0)+np.minimum(0, alpha * (np.exp(X) - 1)),
+        Elu.activation(X, alpha)
+    ) is not True:
+        raise AssertionError
+
+
+def test_Softmax():
+    X = np.array([1.3, 5.1, 2.2, 0.7, 1.1])
+    x_vector = X.reshape(X.shape[0], 1)
+    if np.array_equal(
+        np.exp(X)/np.sum(np.exp(X)),
+        Softmax.activation(X)
+    ) is not True:
+        raise AssertionError
+    x_vector = X.reshape(X.shape[0], 1)
+    x_matrix = np.tile(x_vector, X.shape[0])
+    x_der = np.diag(X) - (x_matrix * np.transpose(x_matrix))
+    if np.array_equal(
+        x_der,
+        Softmax.derivative(X)
+    ) is not True:
+        raise AssertionError
+
+
+def test_Softsign():
+    X = np.array([1.3, 5.1, 2.2, 0.7, 1.1])
+    if np.array_equal(
+        X / (np.abs(X) + 1),
+        Softsign.activation(X)
+    ) is not True:
+        raise AssertionError
+    if np.array_equal(
+        1 / (np.abs(X) + 1)**2,
+        Softsign.derivative(X)
+    ) is not True:
+        raise AssertionError
+
+
+def test_Swish(alpha):
+    X = np.array([1.3, 5.1, 2.2, 0.7, 1.1])
+    if np.array_equal(
+        X / (1 + np.exp(-(alpha*X))),
+        Swish.activation(X)
+    ) is not True:
+        raise AssertionError
+    s = 1 / (1 + np.exp(-X))
+    f = X / (1 + np.exp(-(alpha*X)))
+    df = f + (s * (1 - f))
     if np.array_equal(
-        np.maximum(X,0)+np.minimum(0, alpha * (np.exp(X) - 1)),
-        Elu.activation(X,alpha)
+        df,
+        Swish.derivative(X)
     ) is not True:
         raise AssertionError
-test_Elu(1)