diff --git a/include/cneuron/cneuron.h b/include/cneuron/cneuron.h index dd60ad8b7..3a0b5ac0b 100644 --- a/include/cneuron/cneuron.h +++ b/include/cneuron/cneuron.h @@ -93,10 +93,17 @@ void noise_data(float *data, size_t inputs_length, float noise_factor, float pro * @param a Pointer to the vector. * @param b Pointer to the resulting vector. * @param length Number of element of the vector. - * @param activation_function Activation function used to apply activation. - * @param is_derivative Toggle between derivative calculation and non derivative calculation. */ -void vector_apply_activation(const float *a, float *b, size_t length, float (*activation_function)(float, bool), bool is_derivative); +void vector_apply_activation(const float *a, float *b, size_t length); + +/** + * @brief Apply activation derivative to a vector. + * + * @param a Pointer to the vector. + * @param b Pointer to the resulting vector. + * @param length Number of element of the vector. + */ +void vector_apply_d_activation(const float *a, float *b, size_t length); /** * @brief Compute hadamard product of two vector. @@ -112,18 +119,17 @@ void hadamard_product(const float *restrict a, const float *restrict b, float *r * @brief Represents a neural network with multiple layers. */ typedef struct { - size_t length; /**< Number of layers in the network. */ - size_t inputs_length; /**< Number of inputs to the network. */ - size_t total_allocated_memory; /**< The total memory allocated for the whole neural network. */ - size_t *layer_lengths; /**< Number of neuron in each layer. */ - size_t *prev_lengths_sums; /**< Number of neuron from all previous layer. */ - size_t *prev_weights_sums; /**< Number of weights from all previous layer. */ - float (*activation_function)(float, bool); /**< Pointer to the activation function used in the network. */ - float *delta; /**< Error delta for backpropagation. */ - float *weighted_input; /**< Weighted input values for the layer. */ - float *output; /**< Output values from the layer. */ - float *bias; /**< Bias values for the layer. */ - float *weights; /**< Weights of the layer in column-major format. */ + size_t length; /**< Number of layers in the network. */ + size_t inputs_length; /**< Number of inputs to the network. */ + size_t total_allocated_memory; /**< The total memory allocated for the whole neural network. */ + size_t *layer_lengths; /**< Number of neuron in each layer. */ + size_t *prev_lengths_sums; /**< Number of neuron from all previous layer. */ + size_t *prev_weights_sums; /**< Number of weights from all previous layer. */ + float *delta; /**< Error delta for backpropagation. */ + float *weighted_input; /**< Weighted input values for the layer. */ + float *output; /**< Output values from the layer. */ + float *bias; /**< Bias values for the layer. */ + float *weights; /**< Weights of the layer in column-major format. */ } neural_network; /** @@ -143,11 +149,10 @@ neural_network *alloc_neural_network(size_t network_length, const size_t *layers * @param network_length Number of layers in the network. * @param layers_length Array specifying the number of neurons in each layer. * @param inputs_length Number of inputs to the network. - * @param activation_function Activation function to be used in the network. * * @return Pointer to the newly created neural network. */ -neural_network *get_neural_network(size_t network_length, const size_t *layers_length, size_t inputs_length, float (*activation_function)(float, bool)); +neural_network *get_neural_network(size_t network_length, const size_t *layers_length, size_t inputs_length); /** * @brief Allocates and copy a new neural network. diff --git a/src/linear_algebra.c b/src/linear_algebra.c index ee2890071..7cc4c753f 100644 --- a/src/linear_algebra.c +++ b/src/linear_algebra.c @@ -1,12 +1,21 @@ #include +#include #include #include "cneuron/cneuron.h" -void vector_apply_activation(const float *a, float *b, size_t length, float (*activation_function)(float, bool), bool is_derivative) { - assert(a && b && activation_function); +void vector_apply_activation(const float *a, float *b, size_t length) { + assert(a && b); for (size_t i = 0; i < length; i++) { - b[i] = activation_function(a[i], is_derivative); + b[i] = 1.0f / (1.0f + expf(-a[i])); + } +} + +void vector_apply_d_activation(const float *a, float *b, size_t length) { + assert(a && b); + for (size_t i = 0; i < length; i++) { + float result = 1.0f / (1.0f + expf(-a[i])); + b[i] = result * (1.0f - result); } } diff --git a/src/main.c b/src/main.c index c1b99b652..550eac368 100644 --- a/src/main.c +++ b/src/main.c @@ -23,13 +23,6 @@ float sigmoid(float val, bool is_deravative) { return result; } -float relu(float val, bool is_deravative) { - if (is_deravative) - return (val > 0.0f) ? 1.0f : 0.0f; - - return fmax(0.0f, val); -} - typedef struct { dataset *train_dataset; size_t batch_size; @@ -38,44 +31,59 @@ typedef struct { dataset *dataset_generator(generator_args *args) { dataset *batch_dataset = get_random_dataset_sample(args->train_dataset, args->batch_size); for (size_t i = 0; i < batch_dataset->length; i++) { + int stochastic_choice = rand() % 4; float *data = &batch_dataset->all_inputs[i * batch_dataset->inputs_length]; - rotate_data(data, IMAGE_SIZE, IMAGE_SIZE, randf(10.0f, -5.0f)); - scale_data(data, IMAGE_SIZE, IMAGE_SIZE, randf(1.2f, -0.1f)); - offset_data(data, IMAGE_SIZE, IMAGE_SIZE, randf(6.0f, -3.0f), randf(6.0f, -3.0f)); - noise_data(data, IMAGE_SIZE * IMAGE_SIZE, 0.3f, 0.08f); + switch (stochastic_choice) { + case 0: + rotate_data(data, IMAGE_SIZE, IMAGE_SIZE, randf(10.0f, -5.0f)); + break; + case 1: + scale_data(data, IMAGE_SIZE, IMAGE_SIZE, randf(1.2f, -0.1f)); + break; + case 2: + offset_data(data, IMAGE_SIZE, IMAGE_SIZE, randf(6.0f, -3.0f), randf(6.0f, -3.0f)); + break; + case 3: + noise_data(data, IMAGE_SIZE * IMAGE_SIZE, 0.3f, 0.08f); + break; + } } + return batch_dataset; } -void train(neural_network *nn, dataset *restrict train_dataset, dataset *restrict test_dataset, float learn_rate, int batch_amount, int log_amount, size_t batch_size) { +void train(neural_network *nn, dataset *restrict train_dataset, dataset *restrict test_dataset, float learn_rate, size_t batch_size, unsigned int batch_amount) { #ifdef USE_THREADING pthread_t thread; #endif generator_args args = (generator_args){.train_dataset = train_dataset, .batch_size = batch_size}; clock_t start_time = clock(); dataset *batch_dataset = dataset_generator(&args); - for (int i = 0; i < batch_amount; i++) { - if (i % log_amount == 0 && i != 0) { +#ifdef USE_THREADING + if (batch_amount > 1) { + pthread_create(&thread, NULL, (void *(*)(void *))dataset_generator, &args); + } +#endif + for (unsigned long i = 0; i < batch_amount; i++) { + if (i != 0) { float new_cost = cost(nn, test_dataset, 100); clock_t elapsed_ms = clock() - start_time; float elapsed_s = (float)elapsed_ms / CLOCKS_PER_SEC; - float speed = (float)log_amount * batch_size / elapsed_s; - printf("Learned: %zu, cost: %f, elapsed time: %.2fs, speed: %.2f Data/s\n", i * batch_size, new_cost, elapsed_s, speed); + float speed = (float)batch_size / elapsed_s; + printf("Learned: %zu, Batch: %zu, cost: %f, elapsed time: %.4fs, speed: %.2f Data/s\n", i * batch_size, i, new_cost, elapsed_s, speed); start_time = clock(); - } - + free(batch_dataset); #ifdef USE_THREADING - pthread_create(&thread, NULL, (void *(*)(void *))dataset_generator, &args); - mini_batch_gd(nn, learn_rate, batch_dataset); - free(batch_dataset); - void *result = NULL; - pthread_join(thread, &result); - batch_dataset = (dataset *)result; + pthread_join(thread, (void **)&batch_dataset); + if (i < batch_amount - 1) { + pthread_create(&thread, NULL, (void *(*)(void *))dataset_generator, &args); + } #else - mini_batch_gd(nn, learn_rate, batch_dataset); - free(batch_dataset); - batch_dataset = dataset_generator(&args); + batch_dataset = dataset_generator(&args); #endif + } + + mini_batch_gd(nn, learn_rate, batch_dataset); } // Last dataset not used free(batch_dataset); @@ -133,14 +141,12 @@ int main(int argc, char **argv) { const size_t network_length = 3; const size_t layer_lengths[] = {100, 16, 10}; - neural_network *nn = get_neural_network(network_length, layer_lengths, train_dataset->inputs_length, &sigmoid); + neural_network *nn = get_neural_network(network_length, layer_lengths, train_dataset->inputs_length); // Parameters - const float learn_rate = 10.0f; - const size_t batch_size = 3000; - const int learn_amount = 50000000; - const int batch_amount = learn_amount / batch_size; - const int log_amount = 1000; // Log once reached a number of batch + const float learn_rate = 0.2f; + const size_t batch_size = 1000; + const unsigned int batch_amount = 200000; char cmd[100]; FILE *fp; @@ -169,8 +175,8 @@ int main(int argc, char **argv) { printf("Neural network loaded!\n"); } } else if (cmd[0] == 't') { - train(nn, train_dataset, test_dataset, learn_rate, batch_amount, log_amount, batch_size); - printf("Training completed. Trained for %d times.\n", learn_amount); + train(nn, train_dataset, test_dataset, learn_rate, batch_size, batch_amount); + printf("Training completed. Trained for %d batches.\n", batch_amount); } else if (cmd[0] == 'T') { printf("Testing neural network...\n"); printf("Network is %.2f%% correct!\n", test_network_percent(nn, test_dataset)); diff --git a/src/network.c b/src/network.c index a929f4fb6..579e7f9f9 100644 --- a/src/network.c +++ b/src/network.c @@ -52,7 +52,7 @@ neural_network *alloc_neural_network(size_t network_length, const size_t *layers return nn; } -neural_network *get_neural_network(size_t network_length, const size_t *layers_length, size_t inputs_length, float (*activation_function)(float, bool)) { +neural_network *get_neural_network(size_t network_length, const size_t *layers_length, size_t inputs_length) { assert(layers_length); neural_network *nn = alloc_neural_network(network_length, layers_length, inputs_length); @@ -62,8 +62,6 @@ neural_network *get_neural_network(size_t network_length, const size_t *layers_l // Initialise weights to -1.0f - 1.0f nn->weights[i] = randf(2.0f, -1.0f); } - - nn->activation_function = activation_function; return nn; } @@ -86,9 +84,9 @@ neural_network *copy_neural_network(const neural_network *nn) { void compute_network(const neural_network *restrict nn, const float *restrict inputs) { assert(nn && inputs); - cblas_sgemm(CblasColMajor, CblasNoTrans, CblasNoTrans, nn->layer_lengths[0], 1, nn->inputs_length, 1.0f, nn->weights, nn->layer_lengths[0], inputs, nn->inputs_length, 0.0f, nn->weighted_input, nn->layer_lengths[0]); - cblas_saxpy(nn->layer_lengths[0], 1.0f, nn->bias, 1, nn->weighted_input, 1); - vector_apply_activation(nn->weighted_input, nn->output, nn->layer_lengths[0], nn->activation_function, false); + cblas_scopy(nn->layer_lengths[0], nn->bias, 1, nn->weighted_input, 1); + cblas_sgemv(CblasColMajor, CblasNoTrans, nn->layer_lengths[0], nn->inputs_length, 1.0f, nn->weights, nn->layer_lengths[0], inputs, 1, 1.0f, nn->weighted_input, 1); + vector_apply_activation(nn->weighted_input, nn->output, nn->layer_lengths[0]); for (size_t i = 1; i < nn->length; i++) { size_t len = nn->layer_lengths[i]; size_t prev_len = nn->layer_lengths[i - 1]; @@ -96,9 +94,9 @@ void compute_network(const neural_network *restrict nn, const float *restrict in size_t l_sum = nn->prev_lengths_sums[i]; size_t prev_l_sum = nn->prev_lengths_sums[i - 1]; - cblas_sgemm(CblasColMajor, CblasNoTrans, CblasNoTrans, len, 1, prev_len, 1.0f, &nn->weights[w_sum], len, &nn->output[prev_l_sum], prev_len, 0.0f, &nn->weighted_input[l_sum], len); - cblas_saxpy(len, 1.0f, &nn->bias[l_sum], 1, &nn->weighted_input[l_sum], 1); - vector_apply_activation(&nn->weighted_input[l_sum], &nn->output[l_sum], len, nn->activation_function, false); + cblas_scopy(len, &nn->bias[l_sum], 1, &nn->weighted_input[l_sum], 1); + cblas_sgemv(CblasColMajor, CblasNoTrans, len, prev_len, 1.0f, &nn->weights[w_sum], len, &nn->output[prev_l_sum], 1, 1.0f, &nn->weighted_input[l_sum], 1); + vector_apply_activation(&nn->weighted_input[l_sum], &nn->output[l_sum], len); } } @@ -203,7 +201,7 @@ void layer_learn(const neural_network *nn, size_t layer_index, float learn_rate, size_t l_sum = nn->prev_lengths_sums[layer_index]; size_t w_sum = nn->prev_weights_sums[layer_index]; // f'(Z_i) in weighted_input - vector_apply_activation(&nn->weighted_input[l_sum], &nn->weighted_input[l_sum], len, nn->activation_function, true); + vector_apply_d_activation(&nn->weighted_input[l_sum], &nn->weighted_input[l_sum], len); if (layer_index == nn->length - 1) { // Error in output nn->output[l_sum + data_expected_index] -= 1.0f; @@ -242,7 +240,7 @@ void layer_learn_collect_gradient(const neural_network *nn, float *restrict laye size_t len = nn->layer_lengths[layer_index]; size_t l_sum = nn->prev_lengths_sums[layer_index]; // f'(Z_i) in weighted_input - vector_apply_activation(&nn->weighted_input[l_sum], &nn->weighted_input[l_sum], len, nn->activation_function, true); + vector_apply_d_activation(&nn->weighted_input[l_sum], &nn->weighted_input[l_sum], len); if (layer_index == nn->length - 1) { // Error in output nn->output[l_sum + data_expected_index] -= 1.0f; @@ -309,7 +307,7 @@ void *thread_worker(void *arg) { return NULL; } -#define THREAD_COUNT 4 +#define THREAD_COUNT 5 void mini_batch_gd(neural_network *nn, float learn_rate, const dataset *data_batch) { assert(nn && data_batch); diff --git a/test/linear_algebra.cpp b/test/linear_algebra.cpp index 6bf65c854..078344512 100644 --- a/test/linear_algebra.cpp +++ b/test/linear_algebra.cpp @@ -16,7 +16,7 @@ TEST(LinearAlgebraTest, VectorApplyActivation) { a[1] = -1.2f; a[2] = -0.2f; - vector_apply_activation(a, b, length, sigmoid, false); + vector_apply_activation(a, b, length); for (size_t i = 0; i < length; i++) { ASSERT_FLOAT_EQ(b[i], sigmoid(a[i], false)); diff --git a/test/network.cpp b/test/network.cpp index 5734dbf1e..96712f07b 100644 --- a/test/network.cpp +++ b/test/network.cpp @@ -25,12 +25,11 @@ TEST(NetworkTest, GetNeuralNetwork) { const size_t layer_lengths[] = {2, 3, 4}; const size_t inputs_length = 2; - neural_network *nn = get_neural_network(network_length, layer_lengths, inputs_length, &sigmoid); + neural_network *nn = get_neural_network(network_length, layer_lengths, inputs_length); ASSERT_NE(nn, nullptr); EXPECT_EQ(nn->length, network_length); EXPECT_EQ(nn->inputs_length, inputs_length); - EXPECT_EQ(nn->activation_function, &sigmoid); ASSERT_NE(nn->layer_lengths, nullptr); ASSERT_NE(nn->prev_lengths_sums, nullptr); ASSERT_NE(nn->prev_weights_sums, nullptr); @@ -61,7 +60,7 @@ TEST(NetworkTest, FreeDataset) { size_t layer_length = 1; size_t *layer_lengths = (size_t *)malloc(sizeof(size_t) * layer_length); layer_lengths[0] = 2; - neural_network *nn = get_neural_network(layer_length, layer_lengths, 2, nullptr); + neural_network *nn = get_neural_network(layer_length, layer_lengths, 2); free(nn); // No crash @@ -73,7 +72,7 @@ TEST(NetworkTest, ComputeNetwork) { size_t inputs_length = 1; size_t *layer_lengths = (size_t *)malloc(sizeof(size_t) * layer_length); layer_lengths[0] = 1; - neural_network *nn = get_neural_network(layer_length, layer_lengths, inputs_length, &sigmoid); + neural_network *nn = get_neural_network(layer_length, layer_lengths, inputs_length); float *inputs = (float *)malloc(sizeof(float) * inputs_length); inputs[0] = 0.2f; @@ -95,7 +94,7 @@ TEST(NetworkTest, Softmax) { size_t inputs_length = 1; size_t *layer_lengths = (size_t *)malloc(sizeof(size_t) * layer_length); layer_lengths[0] = 3; - neural_network *nn = get_neural_network(layer_length, layer_lengths, inputs_length, &sigmoid); + neural_network *nn = get_neural_network(layer_length, layer_lengths, inputs_length); nn->output[0] = 0.2f; nn->output[1] = 0.3f; @@ -117,7 +116,7 @@ TEST(NetworkTest, StochasticGDSingleLayer) { size_t layer_length = 1; size_t *layer_lengths = (size_t *)malloc(sizeof(size_t) * layer_length); layer_lengths[0] = 2; - neural_network *nn = get_neural_network(layer_length, layer_lengths, test_dataset->inputs_length, &sigmoid); + neural_network *nn = get_neural_network(layer_length, layer_lengths, test_dataset->inputs_length); for (size_t i = 0; i < 50000; i++) { for (size_t j = 0; j < test_dataset->length; j++) { @@ -145,7 +144,7 @@ TEST(NetworkTest, StochasticGDTests) { size_t *layer_lengths = (size_t *)malloc(sizeof(size_t) * layer_length); layer_lengths[0] = 4; layer_lengths[1] = 2; - neural_network *nn = get_neural_network(layer_length, layer_lengths, test_dataset->inputs_length, &sigmoid); + neural_network *nn = get_neural_network(layer_length, layer_lengths, test_dataset->inputs_length); for (size_t i = 0; i < 500000; i++) { for (size_t j = 0; j < test_dataset->length; j++) { @@ -167,7 +166,7 @@ TEST(NetworkTest, StochasticGDTests) { layer_length = 1; layer_lengths = (size_t *)malloc(sizeof(size_t) * layer_length); layer_lengths[0] = 2; - nn = get_neural_network(layer_length, layer_lengths, test_dataset->inputs_length, &sigmoid); + nn = get_neural_network(layer_length, layer_lengths, test_dataset->inputs_length); for (size_t i = 0; i < 50000; i++) { for (size_t j = 0; j < test_dataset->length; j++) { @@ -195,7 +194,7 @@ TEST(NetworkTest, MiniBatchGDTests) { size_t *layer_lengths = (size_t *)malloc(sizeof(size_t) * layer_length); layer_lengths[0] = 4; layer_lengths[1] = 2; - neural_network *nn = get_neural_network(layer_length, layer_lengths, test_dataset->inputs_length, &sigmoid); + neural_network *nn = get_neural_network(layer_length, layer_lengths, test_dataset->inputs_length); for (size_t i = 0; i < 2000000; i++) { dataset *batch_dataset = get_random_dataset_sample(test_dataset, test_dataset->length); @@ -216,7 +215,7 @@ TEST(NetworkTest, MiniBatchGDTests) { layer_length = 1; layer_lengths = (size_t *)malloc(sizeof(size_t) * layer_length); layer_lengths[0] = 2; - nn = get_neural_network(layer_length, layer_lengths, test_dataset->inputs_length, &sigmoid); + nn = get_neural_network(layer_length, layer_lengths, test_dataset->inputs_length); for (size_t i = 0; i < 100000; i++) { dataset *batch_dataset = get_random_dataset_sample(test_dataset, test_dataset->length);