Add dense sigmoid test

test/layers/test_dense.cu

@@ -17,10 +17,13 @@ class DenseLayerTest : public CublasTestFixture {
         std::vector<std::vector<float>>& weights,
         std::vector<float>&              biases,
         float*&                          d_input,
-        float*&                          d_output
+        float*&                          d_output,
+        std::string                      activation
     ) {
         // Create Dense layer
-        Layers::Dense denseLayer(inputSize, outputSize, "linear", cublasHandle);
+        Layers::Dense denseLayer(
+            inputSize, outputSize, activation, cublasHandle
+        );
 
         // Set weights and biases
         denseLayer.setWeights(weights);
@@ -61,7 +64,7 @@ TEST_F(DenseLayerTest, Init) {
             // std::cout << "Dense layer: input size = " << inputSize << ",
             // output size = " << outputSize << std::endl;
             Layers::Dense denseLayer(
-                inputSize, outputSize, "linear", cublasHandle
+                inputSize, outputSize, "sigmoid", cublasHandle
             );
         }
     }
@@ -79,12 +82,12 @@ TEST_F(DenseLayerTest, setWeights) {
         {1.3f, 0.5f, 0.0f, 1.7f}
     };
 
-    Layers::Dense denseLayer(inputSize, outputSize, "linear", cublasHandle);
+    Layers::Dense denseLayer(inputSize, outputSize, "sigmoid", cublasHandle);
 
     denseLayer.setWeights(weights);
 }
 
-TEST_F(DenseLayerTest, ForwardUnitWeightMatrix) {
+TEST_F(DenseLayerTest, ForwardUnitWeightMatrixLinear) {
     int inputSize  = 3;
     int outputSize = 3;
 
@@ -106,7 +109,8 @@ TEST_F(DenseLayerTest, ForwardUnitWeightMatrix) {
     float* d_output;
 
     Layers::Dense denseLayer = commonTestSetup(
-        inputSize, outputSize, input, weights, biases, d_input, d_output
+        inputSize, outputSize, input, weights, biases, d_input, d_output,
+        "linear"
     );
     denseLayer.forward(d_input, d_output);
 
@@ -124,7 +128,7 @@ TEST_F(DenseLayerTest, ForwardUnitWeightMatrix) {
     commonTestTeardown(d_input, d_output);
 }
 
-TEST_F(DenseLayerTest, ForwardRandomWeightMatrix) {
+TEST_F(DenseLayerTest, ForwardRandomWeightMatrixRelu) {
     int inputSize  = 5;
     int outputSize = 4;
 
@@ -142,7 +146,7 @@ TEST_F(DenseLayerTest, ForwardRandomWeightMatrix) {
     float* d_output;
 
     Layers::Dense denseLayer = commonTestSetup(
-        inputSize, outputSize, input, weights, biases, d_input, d_output
+        inputSize, outputSize, input, weights, biases, d_input, d_output, "relu"
     );
 
     denseLayer.forward(d_input, d_output);
@@ -153,7 +157,10 @@ TEST_F(DenseLayerTest, ForwardRandomWeightMatrix) {
     );
     EXPECT_EQ(cublasStatus, CUBLAS_STATUS_SUCCESS);
 
-    std::vector<float> expectedOutput = {10.4f, 13.0f, 8.9f, 9.3f};
+    // weights * inputs = 13.1, 17.5, 8.3, 14.8
+    // + biases = 13.3, 18, 9, 15.9
+
+    std::vector<float> expectedOutput = {13.3f, 18.0f, 9.0f, 15.9f};
     for (int i = 0; i < outputSize; ++i) {
         EXPECT_NEAR(
             output[i], expectedOutput[i], 1e-4
@@ -162,3 +169,50 @@ TEST_F(DenseLayerTest, ForwardRandomWeightMatrix) {
 
     commonTestTeardown(d_input, d_output);
 }
+
+TEST_F(DenseLayerTest, ForwardRandomWeightMatrixSigmoid) {
+    int inputSize  = 5;
+    int outputSize = 4;
+
+    std::vector<float> input = {0.1f, 0.2f, 0.3f, 0.4f, 0.5f};
+
+    std::vector<std::vector<float>> weights = {
+        {0.8f, 0.7f, 0.7f, 0.3f, 0.8f},
+        {0.1f, 0.4f, 0.8f, 0.0f, 0.2f},
+        {0.2f, 0.5f, 0.7f, 0.3f, 0.0f},
+        {0.1f, 0.7f, 0.6f, 1.0f, 0.4f}
+    };
+    std::vector<float> biases = {0.1f, 0.2f, 0.3f, 0.4f};
+
+    float* d_input;
+    float* d_output;
+
+    Layers::Dense denseLayer = commonTestSetup(
+        inputSize, outputSize, input, weights, biases, d_input, d_output,
+        "sigmoid"
+    );
+
+    denseLayer.forward(d_input, d_output);
+
+    std::vector<float> output(outputSize);
+    cublasStatus = cublasGetVector(
+        outputSize, sizeof(float), d_output, 1, output.data(), 1
+    );
+    EXPECT_EQ(cublasStatus, CUBLAS_STATUS_SUCCESS);
+
+    // weights * input = 0.95, 0.43, 0.45, 0.93
+    // + biases = 1.05, 0.63, 0.75, 1.33
+    // sigmoid = 0.740775, 0.652489, 0.679179, 0.790841
+
+    std::vector<float> expectedOutput = {
+        0.740775f, 0.652489f, 0.679179f, 0.790841f
+    };
+
+    for (int i = 0; i < outputSize; ++i) {
+        EXPECT_NEAR(
+            output[i], expectedOutput[i], 1e-5
+        );
+    }
+
+    commonTestTeardown(d_input, d_output);
+}