Abstract activation and implement softmax

include/layers/activation.cuh

@@ -0,0 +1,55 @@
+#ifndef CUDANET_ACTIVATION_H
+#define CUDANET_ACTIVATION_H
+
+namespace CUDANet::Layers {
+
+/**
+ * @brief Activation functions
+ * 
+ * SIGMOID: Sigmoid
+ * RELU: Rectified Linear Unit
+ * SOFTMAX: Softmax
+ *
+ */
+enum ActivationType { SIGMOID, RELU, SOFTMAX, NONE };
+
+class Activation {
+  public:
+
+    Activation() = default;
+
+    /**
+     * @brief Construct a new Activation object
+     * 
+     * @param activation Type of activation
+     * @param length     Length of the input
+     */
+    Activation(ActivationType activation, const unsigned int length);
+
+    /**
+     * @brief Destroy the Activation object
+     * 
+     */
+    ~Activation();
+
+    /**
+     * @brief Run the activation function on the input
+     * 
+     * @param d_input Pointer to the input vector on the device
+     */
+    void activate(float* d_input);
+
+
+  private:
+    ActivationType activationType;
+    unsigned int length;
+    unsigned int gridSize;
+
+    float* d_softmax_sum;
+
+};
+
+
+}  // namespace CUDANet::Layers
+
+#endif  // CUDANET_ACTIVATION_H

include/layers/conv2d.cuh

@@ -4,7 +4,7 @@
 #include <string>
 #include <vector>
 
-#include "activations.cuh"
+#include "activation.cuh"
 #include "convolution.cuh"
 #include "ilayer.cuh"
 
@@ -23,18 +23,18 @@ class Conv2d : public ILayer {
      * @param inputChannels Number of channels in the input matrix
      * @param kernelSize Width and height of the convolution kernel
      * @param stride Convolution stride
-     * @param padding Padding type ('SAME' or 'VALID')
      * @param numFilters Number of output filters
-     * @param activation Activation function ('RELU', 'SIGMOID' or 'NONE')
+     * @param padding Padding type ('SAME' or 'VALID')
+     * @param activationType Activation function type ('RELU', 'SIGMOID', 'SOFTMAX' or 'NONE')
      */
     Conv2d(
-        int                inputSize,
-        int                inputChannels,
-        int                kernelSize,
-        int                stride,
-        Layers::Padding    padding,
-        int                numFilters,
-        Layers::Activation activation
+        int                    inputSize,
+        int                    inputChannels,
+        int                    kernelSize,
+        int                    stride,
+        int                    numFilters,
+        Layers::Padding        padding,
+        Layers::ActivationType activationType
     );
 
     /**
@@ -67,17 +67,21 @@ class Conv2d : public ILayer {
 
     /**
      * @brief Get the output width (/ height) of the layer
-     * 
-     * @return int 
+     *
+     * @return int
      */
-    int getOutputSize() { return outputSize; }
+    int getOutputSize() {
+        return outputSize;
+    }
 
     /**
      * @brief Get the padding size of the layer
-     * 
-     * @return int 
+     *
+     * @return int
      */
-    int getPaddingSize() { return paddingSize; }
+    int getPaddingSize() {
+        return paddingSize;
+    }
 
   private:
     // Inputs

include/layers/dense.cuh

@@ -20,9 +20,9 @@ class Dense : public ILayer {
      *
      * @param inputSize Size of the input vector
      * @param outputSize Size of the output vector
-     * @param activation Activation function ('RELU', 'SIGMOID' or 'NONE')
+     * @param activationType Activation function type ('RELU', 'SIGMOID', 'SOFTMAX' or 'NONE')
      */
-    Dense(int inputSize, int outputSize, Layers::Activation activation);
+    Dense(int inputSize, int outputSize, Layers::ActivationType activationType);
 
     /**
      * @brief Destroy the Dense layer

include/layers/ilayer.cuh

@@ -6,15 +6,6 @@
 
 namespace CUDANet::Layers {
 
-/**
- * @brief Activation functions
- * 
- * SIGMOID: Sigmoid
- * RELU: Rectified Linear Unit
- *
- */
-enum Activation { SIGMOID, RELU, NONE };
-
 /**
  * @brief Padding types
  * 
@@ -85,7 +76,6 @@ class ILayer {
     std::vector<float> weights;
     std::vector<float> biases;
 
-    Layers::Activation activation;
 };
 
 }  // namespace CUDANet::Layers