From 13d3d38b68630242a0c02f8ed8814fa2a8c68b11 Mon Sep 17 00:00:00 2001 From: LordMathis Date: Wed, 26 Nov 2025 00:19:33 +0100 Subject: [PATCH] Add dtype parameter to layer constructors --- include/layer.hpp | 4 ++- include/layers/activation.hpp | 5 +-- include/layers/add.hpp | 3 ++ include/layers/avg_pool.hpp | 11 ++++++- include/layers/batch_norm.hpp | 3 +- include/layers/concat.hpp | 3 ++ include/layers/conv2d.hpp | 10 +++++- include/layers/dense.hpp | 3 +- include/layers/max_pool.hpp | 10 +++++- src/layers/activation.cpp | 61 ++++++++++++++++++++++------------- src/layers/add.cpp | 8 +++-- src/layers/avg_pooling.cpp | 25 ++++++++++++-- src/layers/batch_norm.cpp | 22 +++++++++---- src/layers/concat.cpp | 7 ++-- src/layers/conv2d.cpp | 18 +++++++++-- src/layers/dense.cpp | 11 +++++-- src/layers/max_pool.cpp | 14 +++++++- 17 files changed, 169 insertions(+), 49 deletions(-) diff --git a/include/layer.hpp b/include/layer.hpp index 1c018b5..d27c18c 100644 --- a/include/layer.hpp +++ b/include/layer.hpp @@ -16,6 +16,8 @@ namespace CUDANet { * */ class Layer { + protected: + CUDANet::DType dtype; public: virtual ~Layer(){}; @@ -39,4 +41,4 @@ class Layer { virtual size_t get_biases_size() = 0; }; -} // namespace CUDANet::Layers +} // namespace CUDANet diff --git a/include/layers/activation.hpp b/include/layers/activation.hpp index 40755fc..78fd82e 100644 --- a/include/layers/activation.hpp +++ b/include/layers/activation.hpp @@ -20,12 +20,13 @@ enum ActivationType { SIGMOID, RELU, SOFTMAX, NONE }; * @brief Utility class that performs activation * */ -class Activation : public Layer { +class Activation : public CUDANet::Layer { public: Activation() = default; Activation(ActivationType activation, const CUDANet::Shape &shape, CUDANet::Backend* backend); + Activation(ActivationType activation, const CUDANet::Shape &shape, CUDANet::DType dtype, CUDANet::Backend* backend); ~Activation() = default; @@ -50,7 +51,7 @@ class Activation : public Layer { private: CUDANet::Backend* backend; - ActivationType activationType; + ActivationType activation_type; CUDANet::Shape shape; CUDANet::Tensor softmax_sum; diff --git a/include/layers/add.hpp b/include/layers/add.hpp index 72aa849..0b0da1e 100644 --- a/include/layers/add.hpp +++ b/include/layers/add.hpp @@ -8,6 +8,7 @@ namespace CUDANet::Layers { class Add { public: Add(CUDANet::Shape a_shape, CUDANet::Shape b_shape, CUDANet::Backend* backend); + Add(CUDANet::Shape a_shape, CUDANet::Shape b_shape, CUDANet::DType dtype, CUDANet::Backend* backend); ~Add(); @@ -19,6 +20,8 @@ class Add { CUDANet::Tensor output; CUDANet::Backend *backend; + + CUDANet::DType dtype; }; } // namespace CUDANet::Layers diff --git a/include/layers/avg_pool.hpp b/include/layers/avg_pool.hpp index a8a9cc5..5648790 100644 --- a/include/layers/avg_pool.hpp +++ b/include/layers/avg_pool.hpp @@ -4,7 +4,7 @@ namespace CUDANet::Layers { -class AvgPool2d : public Layer { +class AvgPool2d : public CUDANet::Layer { public: AvgPool2d( CUDANet::Shape input_shape, @@ -13,6 +13,14 @@ class AvgPool2d : public Layer { CUDANet::Shape padding_shape, CUDANet::Backend *backend ); + AvgPool2d( + CUDANet::Shape input_shape, + CUDANet::Shape pool_shape, + CUDANet::Shape stride_shape, + CUDANet::Shape padding_shape, + CUDANet::DType dtype, + CUDANet::Backend *backend + ); ~AvgPool2d(); @@ -50,6 +58,7 @@ class AvgPool2d : public Layer { class AdaptiveAvgPool2d : public AvgPool2d { public: AdaptiveAvgPool2d(CUDANet::Shape input_shape, CUDANet::Shape output_shape, CUDANet::Backend *backend); + AdaptiveAvgPool2d(CUDANet::Shape input_shape, CUDANet::Shape output_shape, CUDANet::DType dtype, CUDANet::Backend *backend); }; } // namespace CUDANet::Layers diff --git a/include/layers/batch_norm.hpp b/include/layers/batch_norm.hpp index 1b605e9..ddad4da 100644 --- a/include/layers/batch_norm.hpp +++ b/include/layers/batch_norm.hpp @@ -4,9 +4,10 @@ namespace CUDANet::Layers { -class BatchNorm2d : public Layer { +class BatchNorm2d : public CUDANet::Layer { public: BatchNorm2d(CUDANet::Shape input_shape, float epsilon, CUDANet::Backend *backend); + BatchNorm2d(CUDANet::Shape input_shape, float epsilon, CUDANet::DType dtype, CUDANet::Backend *backend); ~BatchNorm2d(); diff --git a/include/layers/concat.hpp b/include/layers/concat.hpp index 400006c..a225f4f 100644 --- a/include/layers/concat.hpp +++ b/include/layers/concat.hpp @@ -12,6 +12,7 @@ class Concat { public: Concat(const CUDANet::Shape a_shape, const CUDANet::Shape b_shape, CUDANet::Backend *backend); + Concat(const CUDANet::Shape a_shape, const CUDANet::Shape b_shape, CUDANet::DType dtype, CUDANet::Backend *backend); ~Concat(); @@ -27,6 +28,8 @@ class Concat { CUDANet::Tensor output; CUDANet::Backend *backend; + + CUDANet::DType dtype; }; } // namespace CUDANet::Layers diff --git a/include/layers/conv2d.hpp b/include/layers/conv2d.hpp index 7218c7e..a742130 100644 --- a/include/layers/conv2d.hpp +++ b/include/layers/conv2d.hpp @@ -8,7 +8,7 @@ namespace CUDANet::Layers { * @brief 2D convolutional layer * */ -class Conv2d : public Layer { +class Conv2d : public CUDANet::Layer { public: Conv2d( CUDANet::Shape input_shape, @@ -17,6 +17,14 @@ class Conv2d : public Layer { CUDANet::Shape padding_shape, CUDANet::Backend* backend ); + Conv2d( + CUDANet::Shape input_shape, + CUDANet::Shape kernel_shape, + CUDANet::Shape stride_shape, + CUDANet::Shape padding_shape, + CUDANet::DType dtype, + CUDANet::Backend* backend + ); ~Conv2d(); diff --git a/include/layers/dense.hpp b/include/layers/dense.hpp index c58d4fb..aa105e3 100644 --- a/include/layers/dense.hpp +++ b/include/layers/dense.hpp @@ -9,10 +9,11 @@ namespace CUDANet::Layers { * @brief Dense (fully connected) layer * */ -class Dense : public Layer { +class Dense : public CUDANet::Layer { public: Dense(CUDANet::Shape input_shape, CUDANet::Shape output_shape, CUDANet::Backend *backend); + Dense(CUDANet::Shape input_shape, CUDANet::Shape output_shape, CUDANet::DType dtype, CUDANet::Backend *backend); ~Dense(); diff --git a/include/layers/max_pool.hpp b/include/layers/max_pool.hpp index f4062d3..962ef42 100644 --- a/include/layers/max_pool.hpp +++ b/include/layers/max_pool.hpp @@ -4,7 +4,7 @@ namespace CUDANet::Layers { -class MaxPool2d : public Layer { +class MaxPool2d : public CUDANet::Layer { public: MaxPool2d( CUDANet::Shape input_shape, @@ -13,6 +13,14 @@ class MaxPool2d : public Layer { CUDANet::Shape padding_shape, CUDANet::Backend* backend ); + MaxPool2d( + CUDANet::Shape input_shape, + CUDANet::Shape pool_shape, + CUDANet::Shape stride_shape, + CUDANet::Shape padding_shape, + CUDANet::DType dtype, + CUDANet::Backend* backend + ); ~MaxPool2d(); CUDANet::Tensor& forward(CUDANet::Tensor &input) override; diff --git a/src/layers/activation.cpp b/src/layers/activation.cpp index 1a70c50..8a4caa2 100644 --- a/src/layers/activation.cpp +++ b/src/layers/activation.cpp @@ -1,41 +1,58 @@ +#include "activation.hpp" + #include #include #include -#include "activation.hpp" #include "tensor.hpp" using namespace CUDANet::Layers; -Activation::Activation(ActivationType activation, const CUDANet::Shape &shape, CUDANet::Backend* backend) - : backend(backend), activationType(activation), shape(shape) { +Activation::Activation( + ActivationType activation, + const CUDANet::Shape& shape, + CUDANet::Backend* backend +) + : Activation(activation, shape, backend->get_default_dtype(), backend) {} +Activation::Activation( + ActivationType activation, + const CUDANet::Shape& shape, + CUDANet::DType dtype, + CUDANet::Backend* backend +) + : activation_type(activation), + shape(shape), + backend(backend) { + this->dtype = dtype; + if (shape.size() != 1) { throw InvalidShapeException("input", 1, shape.size()); } auto length = shape[0]; - if (activationType == SOFTMAX) { - softmax_sum = CUDANet::Tensor({static_cast(length)}, CUDANet::DType::FLOAT32, backend); - tensor_max = CUDANet::Tensor({static_cast(length)}, CUDANet::DType::FLOAT32, backend); + if (activation_type == SOFTMAX) { + softmax_sum = + CUDANet::Tensor({static_cast(length)}, dtype, backend); + tensor_max = + CUDANet::Tensor({static_cast(length)}, dtype, backend); } } -CUDANet::Tensor& Activation::forward(CUDANet::Tensor &input) { - switch (activationType) - { - case ActivationType::SIGMOID: - backend->sigmoid(input); - break; - case ActivationType::RELU: - backend->relu(input); - break; - case ActivationType::SOFTMAX: - backend->softmax(input, tensor_max, softmax_sum); - break; - default: - break; +CUDANet::Tensor& Activation::forward(CUDANet::Tensor& input) { + switch (activation_type) { + case ActivationType::SIGMOID: + backend->sigmoid(input); + break; + case ActivationType::RELU: + backend->relu(input); + break; + case ActivationType::SOFTMAX: + backend->softmax(input, tensor_max, softmax_sum); + break; + default: + break; } return input; @@ -57,13 +74,13 @@ size_t Activation::output_size() { return shape[0]; } -void Activation::set_weights(void *input) {} +void Activation::set_weights(void* input) {} size_t Activation::get_weights_size() { return 0; } -void Activation::set_biases(void *input) {} +void Activation::set_biases(void* input) {} size_t Activation::get_biases_size() { return 0; diff --git a/src/layers/add.cpp b/src/layers/add.cpp index 72206e0..be989bd 100644 --- a/src/layers/add.cpp +++ b/src/layers/add.cpp @@ -3,7 +3,11 @@ using namespace CUDANet::Layers; -Add::Add(CUDANet::Shape a_shape, CUDANet::Shape b_shape, CUDANet::Backend* backend) : backend(backend) { +Add::Add(CUDANet::Shape a_shape, CUDANet::Shape b_shape, CUDANet::Backend* backend) + : Add(a_shape, b_shape, backend->get_default_dtype(), backend) {} + +Add::Add(CUDANet::Shape a_shape, CUDANet::Shape b_shape, CUDANet::DType dtype, CUDANet::Backend* backend) + : backend(backend), dtype(dtype) { if (a_shape != b_shape) { throw InvalidShapeException( "Add requires matching dimensions", a_shape, b_shape @@ -11,7 +15,7 @@ Add::Add(CUDANet::Shape a_shape, CUDANet::Shape b_shape, CUDANet::Backend* backe } out_shape = a_shape; - output = CUDANet::Tensor(out_shape, CUDANet::DType::FLOAT32, backend); + output = CUDANet::Tensor(out_shape, dtype, backend); } Add::~Add() {} diff --git a/src/layers/avg_pooling.cpp b/src/layers/avg_pooling.cpp index 3786cfc..375e2c9 100644 --- a/src/layers/avg_pooling.cpp +++ b/src/layers/avg_pooling.cpp @@ -11,6 +11,16 @@ AvgPool2d::AvgPool2d( CUDANet::Shape stride_shape, CUDANet::Shape padding_shape, CUDANet::Backend* backend +) + : AvgPool2d(input_shape, pool_shape, stride_shape, padding_shape, backend->get_default_dtype(), backend) {} + +AvgPool2d::AvgPool2d( + CUDANet::Shape input_shape, + CUDANet::Shape pool_shape, + CUDANet::Shape stride_shape, + CUDANet::Shape padding_shape, + CUDANet::DType dtype, + CUDANet::Backend* backend ) : in_shape(input_shape), pool_shape(pool_shape), @@ -33,6 +43,8 @@ AvgPool2d::AvgPool2d( throw InvalidShapeException("padding", 2, padding_shape.size()); } + this->dtype = dtype; + out_shape = { (in_shape[0] + 2 * padding_shape[0] - pool_shape[0]) / stride_shape[0] + 1, @@ -43,7 +55,7 @@ AvgPool2d::AvgPool2d( output = CUDANet::Tensor( Shape{out_shape[0] * out_shape[1] * out_shape[2]}, - CUDANet::DType::FLOAT32, backend + dtype, backend ); } @@ -96,6 +108,14 @@ AdaptiveAvgPool2d::AdaptiveAvgPool2d( CUDANet::Shape input_shape, CUDANet::Shape output_shape, CUDANet::Backend *backend +) + : AdaptiveAvgPool2d(input_shape, output_shape, backend->get_default_dtype(), backend) {} + +AdaptiveAvgPool2d::AdaptiveAvgPool2d( + CUDANet::Shape input_shape, + CUDANet::Shape output_shape, + CUDANet::DType dtype, + CUDANet::Backend *backend ) : AvgPool2d( input_shape, @@ -114,12 +134,13 @@ AdaptiveAvgPool2d::AdaptiveAvgPool2d( (input_shape[0] - (output_shape[0] - 1) * (input_shape[0] / output_shape[0]) - 1) / 2, (input_shape[1] - (output_shape[1] - 1) * (input_shape[1] / output_shape[1]) - 1) / 2 }, + dtype, backend ) { out_shape = output_shape; output = CUDANet::Tensor( Shape{out_shape[0] * out_shape[1] * out_shape[2]}, - CUDANet::DType::FLOAT32, backend + dtype, backend ); } diff --git a/src/layers/batch_norm.cpp b/src/layers/batch_norm.cpp index 213c16b..6770f84 100644 --- a/src/layers/batch_norm.cpp +++ b/src/layers/batch_norm.cpp @@ -12,6 +12,14 @@ BatchNorm2d::BatchNorm2d( CUDANet::Shape input_shape, float eps, CUDANet::Backend *backend +) + : BatchNorm2d(input_shape, eps, backend->get_default_dtype(), backend) {} + +BatchNorm2d::BatchNorm2d( + CUDANet::Shape input_shape, + float eps, + CUDANet::DType dtype, + CUDANet::Backend *backend ) : in_shape(input_shape), backend(backend) { @@ -19,22 +27,24 @@ BatchNorm2d::BatchNorm2d( throw InvalidShapeException("input", 3, in_shape.size()); } - epsilon = CUDANet::Tensor({1}, CUDANet::DType::FLOAT32, backend); + this->dtype = dtype; + + epsilon = CUDANet::Tensor({1}, dtype, backend); epsilon.set_data(&eps); - running_mean = CUDANet::Tensor({in_shape[2]}, CUDANet::DType::FLOAT32, backend); + running_mean = CUDANet::Tensor({in_shape[2]}, dtype, backend); running_mean.zero(); - running_var = CUDANet::Tensor({in_shape[2]}, CUDANet::DType::FLOAT32, backend); + running_var = CUDANet::Tensor({in_shape[2]}, dtype, backend); running_var.fill(1); - weights = CUDANet::Tensor({in_shape[2]}, CUDANet::DType::FLOAT32, backend); + weights = CUDANet::Tensor({in_shape[2]}, dtype, backend); weights.fill(1); - biases = CUDANet::Tensor({in_shape[2]}, CUDANet::DType::FLOAT32, backend); + biases = CUDANet::Tensor({in_shape[2]}, dtype, backend); biases.zero(); - output = CUDANet::Tensor(in_shape, CUDANet::DType::FLOAT32, backend); + output = CUDANet::Tensor(in_shape, dtype, backend); } BatchNorm2d::~BatchNorm2d() {} diff --git a/src/layers/concat.cpp b/src/layers/concat.cpp index e4f365d..db1383b 100644 --- a/src/layers/concat.cpp +++ b/src/layers/concat.cpp @@ -3,7 +3,10 @@ using namespace CUDANet::Layers; Concat::Concat(const CUDANet::Shape a_shape, const CUDANet::Shape b_shape, CUDANet::Backend *backend) - : a_shape(a_shape), b_shape(b_shape), backend(backend) { + : Concat(a_shape, b_shape, backend->get_default_dtype(), backend) {} + +Concat::Concat(const CUDANet::Shape a_shape, const CUDANet::Shape b_shape, CUDANet::DType dtype, CUDANet::Backend *backend) + : a_shape(a_shape), b_shape(b_shape), backend(backend), dtype(dtype) { if (a_shape[0] != b_shape[0] || a_shape[1] != b_shape[1]) { throw InvalidShapeException( "Concat requires matching height and width dimensions", a_shape, @@ -12,7 +15,7 @@ Concat::Concat(const CUDANet::Shape a_shape, const CUDANet::Shape b_shape, CUDAN } out_shape = {a_shape[0], a_shape[1], a_shape[2] + b_shape[2]}; - output = CUDANet::Tensor(out_shape, CUDANet::DType::FLOAT32, backend); + output = CUDANet::Tensor(out_shape, dtype, backend); } Concat::~Concat() {} diff --git a/src/layers/conv2d.cpp b/src/layers/conv2d.cpp index 45cd92f..f297ebe 100644 --- a/src/layers/conv2d.cpp +++ b/src/layers/conv2d.cpp @@ -14,6 +14,16 @@ Conv2d::Conv2d( CUDANet::Shape stride_shape, CUDANet::Shape padding_shape, CUDANet::Backend* backend +) + : Conv2d(input_shape, kernel_shape, stride_shape, padding_shape, backend->get_default_dtype(), backend) {} + +Conv2d::Conv2d( + CUDANet::Shape input_shape, + CUDANet::Shape kernel_shape, + CUDANet::Shape stride_shape, + CUDANet::Shape padding_shape, + CUDANet::DType dtype, + CUDANet::Backend* backend ) : in_shape(input_shape), kernel_shape(kernel_shape), @@ -36,6 +46,8 @@ Conv2d::Conv2d( throw InvalidShapeException("padding", 3, padding_shape.size()); } + this->dtype = dtype; + out_shape = { (in_shape[0] - kernel_shape[0] + 2 * padding_shape[0]) / stride_shape[0] + @@ -48,17 +60,17 @@ Conv2d::Conv2d( output = CUDANet::Tensor( Shape{out_shape[0], out_shape[1], out_shape[2]}, - CUDANet::DType::FLOAT32, backend + dtype, backend ); weights = CUDANet::Tensor( Shape{ kernel_shape[0], kernel_shape[1], kernel_shape[2], in_shape[2] }, - CUDANet::DType::FLOAT32, backend + dtype, backend ); biases = CUDANet::Tensor( - Shape{kernel_shape[2]}, CUDANet::DType::FLOAT32, backend + Shape{kernel_shape[2]}, dtype, backend ); weights.zero(); diff --git a/src/layers/dense.cpp b/src/layers/dense.cpp index 3895e57..b273c09 100644 --- a/src/layers/dense.cpp +++ b/src/layers/dense.cpp @@ -6,6 +6,9 @@ using namespace CUDANet::Layers; Dense::Dense(CUDANet::Shape in_shape, CUDANet::Shape out_shape, CUDANet::Backend* backend) + : Dense(in_shape, out_shape, backend->get_default_dtype(), backend) {} + +Dense::Dense(CUDANet::Shape in_shape, CUDANet::Shape out_shape, CUDANet::DType dtype, CUDANet::Backend* backend) : backend(backend), in_shape(in_shape), out_shape(out_shape) { @@ -18,9 +21,11 @@ Dense::Dense(CUDANet::Shape in_shape, CUDANet::Shape out_shape, CUDANet::Backend throw InvalidShapeException("output", 1, out_shape.size()); } - weights = CUDANet::Tensor(Shape{out_shape[0], in_shape[0]}, CUDANet::DType::FLOAT32, backend); - biases = CUDANet::Tensor(Shape{out_shape[0]}, CUDANet::DType::FLOAT32, backend); - output = CUDANet::Tensor(Shape{out_shape[0]}, CUDANet::DType::FLOAT32, backend); + this->dtype = dtype; + + weights = CUDANet::Tensor(Shape{out_shape[0], in_shape[0]}, dtype, backend); + biases = CUDANet::Tensor(Shape{out_shape[0]}, dtype, backend); + output = CUDANet::Tensor(Shape{out_shape[0]}, dtype, backend); weights.zero(); biases.zero(); diff --git a/src/layers/max_pool.cpp b/src/layers/max_pool.cpp index a5ff35b..38ba131 100644 --- a/src/layers/max_pool.cpp +++ b/src/layers/max_pool.cpp @@ -10,6 +10,16 @@ MaxPool2d::MaxPool2d( CUDANet::Shape stride_shape, CUDANet::Shape padding_shape, CUDANet::Backend* backend +) + : MaxPool2d(input_shape, pool_shape, stride_shape, padding_shape, backend->get_default_dtype(), backend) {} + +MaxPool2d::MaxPool2d( + CUDANet::Shape input_shape, + CUDANet::Shape pool_shape, + CUDANet::Shape stride_shape, + CUDANet::Shape padding_shape, + CUDANet::DType dtype, + CUDANet::Backend* backend ) : in_shape(input_shape), pool_shape(pool_shape), @@ -32,6 +42,8 @@ MaxPool2d::MaxPool2d( throw InvalidShapeException("padding", 2, padding_shape.size()); } + this->dtype = dtype; + out_shape = { (in_shape[0] + 2 * padding_shape[0] - pool_shape[0]) / stride_shape[0] + 1, @@ -42,7 +54,7 @@ MaxPool2d::MaxPool2d( output = CUDANet::Tensor( Shape{out_shape[0] * out_shape[1] * out_shape[2]}, - CUDANet::DType::FLOAT32, backend + dtype, backend ); }