Mathis/CUDANet
1
0
Fork 0
mirror of https://github.com/lordmathis/CUDANet.git synced 2025-11-07 02:04:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve softmax numerical stability

Browse Source
This commit is contained in:
Matúš Námešný
2024-04-08 23:25:46 +02:00
parent e419a93408
commit b49dddf34a
6 changed files with 119 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
test/kernels/test_matmul.cu
View File

@@ -3,6 +3,7 @@
#include <vector>
#include "cuda_helper.cuh"
#include "matmul.cuh"
TEST(MatMulTest, MatVecMulTest) {
@@ -61,3 +62,31 @@ TEST(MatMulTest, MatVecMulTest) {
}
}
TEST(MatMulTest, MaxReduceTest) {
cudaError_t cudaStatus;
std::vector<float> input = {0.643f, 0.912f, 0.723f, 0.587f, 0.155f, 0.932f, 0.391f, 0.279f, 0.846f, 0.788f};
float* d_input;
float* d_output;
cudaStatus = cudaMalloc((void**)&d_input, sizeof(float) * 10);
EXPECT_EQ(cudaStatus, cudaSuccess);
cudaStatus = cudaMalloc((void**)&d_output, sizeof(float));
EXPECT_EQ(cudaStatus, cudaSuccess);
cudaStatus = cudaMemcpy(d_input, input.data(), sizeof(float) * 10, cudaMemcpyHostToDevice);
EXPECT_EQ(cudaStatus, cudaSuccess);
const int grid_size = (10 + BLOCK_SIZE - 1) / BLOCK_SIZE;
CUDANet::Kernels::max_reduce<<<grid_size, BLOCK_SIZE>>>(d_input, d_output);
CUDANet::Kernels::max_reduce<<<1, BLOCK_SIZE>>>(d_output, d_output);
std::vector<float> output(10);
cudaStatus = cudaMemcpy(output.data(), d_output, sizeof(float), cudaMemcpyDeviceToHost);
EXPECT_EQ(cudaStatus, cudaSuccess);
EXPECT_EQ(output[0], 0.932f);
}

9
test/model/test_model.cu
View File

@@ -52,7 +52,7 @@ TEST(Model, TestModelPredict) {
// dense
CUDANet::Layers::Dense dense(
18, 6, CUDANet::Layers::ActivationType::NONE
18, 6, CUDANet::Layers::ActivationType::SOFTMAX
);
// dense weights 18*6
std::vector<float> denseWeights = {
@@ -93,13 +93,14 @@ TEST(Model, TestModelPredict) {
// predict
const float* output = model.predict(input.data());
float sum = 0.0f;
// float sum = 0.0f;
for (int i = 0; i < outputSize; ++i) {
// sum += output[i];
sum += output[i];
std::cout << output[i] << " ";
}
// EXPECT_NEAR(sum, 1.0f, 1e-5f);
std::cout << std::endl;
EXPECT_NEAR(sum, 1.0f, 1e-5f);
}