Mathis/CUDANet
1
0
Fork 0
mirror of https://github.com/lordmathis/CUDANet.git synced 2025-11-06 09:44:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Move softmax partial kernels to matmul

Browse Source
This commit is contained in:
Matúš Námešný
2024-04-11 22:01:47 +02:00
parent bf7c961b9e
commit 710a33bdde
6 changed files with 274 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
test/kernels/test_activation_functions.cu
View File

@@ -4,6 +4,7 @@
#include <iostream>
#include "activation_functions.cuh"
#include "matmul.cuh"
#include "cuda_helper.cuh"
TEST(ActivationFunctionsTest, SigmoidSanityCheck) {
@@ -43,93 +44,24 @@ TEST(ActivationFunctionsTest, SigmoidSanityCheck) {
cudaFree(d_input);
cudaFree(d_output);
cudaDeviceReset();
}
TEST(ActivationFunctionsTest, SoftmaxExpTest) {
cudaError_t cudaStatus;
// void print_vec(float* d_vec, int length) {
float input[6] = {22.496f, 36.9006f, 30.9904f,
28.4213f, 26.4541f, 31.7887f};
// std::vector<float> h_vec(length);
// CUDA_CHECK(cudaMemcpy(
// h_vec.data(), d_vec, sizeof(float) * length, cudaMemcpyDeviceToHost
// ));
std::vector<float> expected = {5886928896.0f, 1.06102872080384e+16f,
28771323215872.0f, 2204012904448.0f,
308226162688.0f, 63922983927808.0f};
// float sum = 0.0f;
float* d_input;
float* d_output;
// for (int i = 0; i < length; ++i) {
// std::cout << h_vec[i] << ", ";
// sum += h_vec[i];
// }
cudaStatus = cudaMalloc((void**)&d_input, sizeof(float) * 6);
EXPECT_EQ(cudaStatus, cudaSuccess);
// std::cout << std::endl;
cudaStatus = cudaMalloc((void**)&d_output, sizeof(float) * 6);
EXPECT_EQ(cudaStatus, cudaSuccess);
cudaStatus =
cudaMemcpy(d_input, input, sizeof(float) * 6, cudaMemcpyHostToDevice);
EXPECT_EQ(cudaStatus, cudaSuccess);
CUDANet::Kernels::softmax_exp<<<1, 6>>>(d_input, d_output, 6);
cudaStatus = cudaDeviceSynchronize();
EXPECT_EQ(cudaStatus, cudaSuccess);
std::vector<float> output(6);
cudaStatus = cudaMemcpy(
output.data(), d_output, sizeof(float) * 6, cudaMemcpyDeviceToHost
);
EXPECT_EQ(cudaStatus, cudaSuccess);
for (int i = 0; i < 6; i++) {
EXPECT_NEAR(expected[i], output[i], 1e7);
}
cudaFree(d_input);
cudaFree(d_output);
}
TEST(ActivationFunctionsTest, SoftmaxSumTest) {
cudaError_t cudaStatus;
const int n = 10;
std::vector<float> input(n);
for (int i = 0; i < n; i++) {
input[i] = i;
}
const float expected = n * (n - 1) / 2;
float* d_input;
float* d_sum;
const int gridSize = (n + BLOCK_SIZE - 1) / BLOCK_SIZE;
cudaStatus = cudaMalloc((void**)&d_input, sizeof(float) * n);
EXPECT_EQ(cudaStatus, cudaSuccess);
cudaStatus = cudaMalloc((void**)&d_sum, sizeof(float) * n);
EXPECT_EQ(cudaStatus, cudaSuccess);
cudaStatus =
cudaMemcpy(d_input, input.data(), sizeof(float) * n, cudaMemcpyHostToDevice);
EXPECT_EQ(cudaStatus, cudaSuccess);
CUDANet::Kernels::softmax_sum<<<gridSize, BLOCK_SIZE>>>(
d_input, d_sum
);
CUDANet::Kernels::softmax_sum<<<1, BLOCK_SIZE>>>(
d_sum, d_sum
);
CUDANet::Kernels::softmax_sum<<<1, BLOCK_SIZE>>>(
d_sum, d_sum
);
std::vector<float> sum(n);
cudaStatus = cudaMemcpy(
sum.data(), d_sum, sizeof(float) * n, cudaMemcpyDeviceToHost
);
EXPECT_EQ(cudaStatus, cudaSuccess);
EXPECT_FLOAT_EQ(expected, sum[0]);
}
// }