Refactor main function to allow batch to run faster

include/cneuron/cneuron.h

@@ -93,10 +93,17 @@ void noise_data(float *data, size_t inputs_length, float noise_factor, float pro
  * @param a Pointer to the vector.
  * @param b Pointer to the resulting vector.
  * @param length Number of element of the vector.
- * @param activation_function Activation function used to apply activation.
- * @param is_derivative Toggle between derivative calculation and non derivative calculation.
  */
-void vector_apply_activation(const float *a, float *b, size_t length, float (*activation_function)(float, bool), bool is_derivative);
+void vector_apply_activation(const float *a, float *b, size_t length);
+
+/**
+ * @brief Apply activation derivative to a vector.
+ *
+ * @param a Pointer to the vector.
+ * @param b Pointer to the resulting vector.
+ * @param length Number of element of the vector.
+ */
+void vector_apply_d_activation(const float *a, float *b, size_t length);
 
 /**
  * @brief Compute hadamard product of two vector.
@@ -112,18 +119,17 @@ void hadamard_product(const float *restrict a, const float *restrict b, float *r
  * @brief Represents a neural network with multiple layers.
  */
 typedef struct {
-    size_t length;                             /**< Number of layers in the network. */
-    size_t inputs_length;                      /**< Number of inputs to the network. */
-    size_t total_allocated_memory;             /**< The total memory allocated for the whole neural network. */
-    size_t *layer_lengths;                     /**< Number of neuron in each layer. */
-    size_t *prev_lengths_sums;                 /**< Number of neuron from all previous layer. */
-    size_t *prev_weights_sums;                 /**< Number of weights from all previous layer. */
-    float (*activation_function)(float, bool); /**< Pointer to the activation function used in the network. */
-    float *delta;                              /**< Error delta for backpropagation. */
-    float *weighted_input;                     /**< Weighted input values for the layer. */
-    float *output;                             /**< Output values from the layer. */
-    float *bias;                               /**< Bias values for the layer. */
-    float *weights;                            /**< Weights of the layer in column-major format. */
+    size_t length;                 /**< Number of layers in the network. */
+    size_t inputs_length;          /**< Number of inputs to the network. */
+    size_t total_allocated_memory; /**< The total memory allocated for the whole neural network. */
+    size_t *layer_lengths;         /**< Number of neuron in each layer. */
+    size_t *prev_lengths_sums;     /**< Number of neuron from all previous layer. */
+    size_t *prev_weights_sums;     /**< Number of weights from all previous layer. */
+    float *delta;                  /**< Error delta for backpropagation. */
+    float *weighted_input;         /**< Weighted input values for the layer. */
+    float *output;                 /**< Output values from the layer. */
+    float *bias;                   /**< Bias values for the layer. */
+    float *weights;                /**< Weights of the layer in column-major format. */
 } neural_network;
 
 /**
@@ -143,11 +149,10 @@ neural_network *alloc_neural_network(size_t network_length, const size_t *layers
  * @param network_length Number of layers in the network.
  * @param layers_length Array specifying the number of neurons in each layer.
  * @param inputs_length Number of inputs to the network.
- * @param activation_function Activation function to be used in the network.
  *
  * @return Pointer to the newly created neural network.
  */
-neural_network *get_neural_network(size_t network_length, const size_t *layers_length, size_t inputs_length, float (*activation_function)(float, bool));
+neural_network *get_neural_network(size_t network_length, const size_t *layers_length, size_t inputs_length);
 
 /**
  * @brief Allocates and copy a new neural network.

src/linear_algebra.c

@@ -1,12 +1,21 @@
 #include <assert.h>
+#include <math.h>
 #include <stdbool.h>
 
 #include "cneuron/cneuron.h"
 
-void vector_apply_activation(const float *a, float *b, size_t length, float (*activation_function)(float, bool), bool is_derivative) {
-    assert(a && b && activation_function);
+void vector_apply_activation(const float *a, float *b, size_t length) {
+    assert(a && b);
     for (size_t i = 0; i < length; i++) {
-        b[i] = activation_function(a[i], is_derivative);
+        b[i] = 1.0f / (1.0f + expf(-a[i]));
+    }
+}
+
+void vector_apply_d_activation(const float *a, float *b, size_t length) {
+    assert(a && b);
+    for (size_t i = 0; i < length; i++) {
+        float result = 1.0f / (1.0f + expf(-a[i]));
+        b[i] = result * (1.0f - result);
     }
 }
 

src/main.c

@@ -23,13 +23,6 @@ float sigmoid(float val, bool is_deravative) {
     return result;
 }
 
-float relu(float val, bool is_deravative) {
-    if (is_deravative)
-        return (val > 0.0f) ? 1.0f : 0.0f;
-
-    return fmax(0.0f, val);
-}
-
 typedef struct {
     dataset *train_dataset;
     size_t batch_size;
@@ -38,44 +31,59 @@ typedef struct {
 dataset *dataset_generator(generator_args *args) {
     dataset *batch_dataset = get_random_dataset_sample(args->train_dataset, args->batch_size);
     for (size_t i = 0; i < batch_dataset->length; i++) {
+        int stochastic_choice = rand() % 4;
         float *data = &batch_dataset->all_inputs[i * batch_dataset->inputs_length];
-        rotate_data(data, IMAGE_SIZE, IMAGE_SIZE, randf(10.0f, -5.0f));
-        scale_data(data, IMAGE_SIZE, IMAGE_SIZE, randf(1.2f, -0.1f));
-        offset_data(data, IMAGE_SIZE, IMAGE_SIZE, randf(6.0f, -3.0f), randf(6.0f, -3.0f));
-        noise_data(data, IMAGE_SIZE * IMAGE_SIZE, 0.3f, 0.08f);
+        switch (stochastic_choice) {
+            case 0:
+                rotate_data(data, IMAGE_SIZE, IMAGE_SIZE, randf(10.0f, -5.0f));
+                break;
+            case 1:
+                scale_data(data, IMAGE_SIZE, IMAGE_SIZE, randf(1.2f, -0.1f));
+                break;
+            case 2:
+                offset_data(data, IMAGE_SIZE, IMAGE_SIZE, randf(6.0f, -3.0f), randf(6.0f, -3.0f));
+                break;
+            case 3:
+                noise_data(data, IMAGE_SIZE * IMAGE_SIZE, 0.3f, 0.08f);
+                break;
+        }
     }
+
     return batch_dataset;
 }
 
-void train(neural_network *nn, dataset *restrict train_dataset, dataset *restrict test_dataset, float learn_rate, int batch_amount, int log_amount, size_t batch_size) {
+void train(neural_network *nn, dataset *restrict train_dataset, dataset *restrict test_dataset, float learn_rate, size_t batch_size, unsigned int batch_amount) {
 #ifdef USE_THREADING
     pthread_t thread;
 #endif
     generator_args args = (generator_args){.train_dataset = train_dataset, .batch_size = batch_size};
     clock_t start_time = clock();
     dataset *batch_dataset = dataset_generator(&args);
-    for (int i = 0; i < batch_amount; i++) {
-        if (i % log_amount == 0 && i != 0) {
+#ifdef USE_THREADING
+    if (batch_amount > 1) {
+        pthread_create(&thread, NULL, (void *(*)(void *))dataset_generator, &args);
+    }
+#endif
+    for (unsigned long i = 0; i < batch_amount; i++) {
+        if (i != 0) {
             float new_cost = cost(nn, test_dataset, 100);
             clock_t elapsed_ms = clock() - start_time;
             float elapsed_s = (float)elapsed_ms / CLOCKS_PER_SEC;
-            float speed = (float)log_amount * batch_size / elapsed_s;
-            printf("Learned: %zu, cost: %f, elapsed time: %.2fs, speed: %.2f Data/s\n", i * batch_size, new_cost, elapsed_s, speed);
+            float speed = (float)batch_size / elapsed_s;
+            printf("Learned: %zu, Batch: %zu, cost: %f, elapsed time: %.4fs, speed: %.2f Data/s\n", i * batch_size, i, new_cost, elapsed_s, speed);
             start_time = clock();
-        }
-
+            free(batch_dataset);
 #ifdef USE_THREADING
-        pthread_create(&thread, NULL, (void *(*)(void *))dataset_generator, &args);
-        mini_batch_gd(nn, learn_rate, batch_dataset);
-        free(batch_dataset);
-        void *result = NULL;
-        pthread_join(thread, &result);
-        batch_dataset = (dataset *)result;
+            pthread_join(thread, (void **)&batch_dataset);
+            if (i < batch_amount - 1) {
+                pthread_create(&thread, NULL, (void *(*)(void *))dataset_generator, &args);
+            }
 #else
-        mini_batch_gd(nn, learn_rate, batch_dataset);
-        free(batch_dataset);
-        batch_dataset = dataset_generator(&args);
+            batch_dataset = dataset_generator(&args);
 #endif
+        }
+
+        mini_batch_gd(nn, learn_rate, batch_dataset);
     }
     // Last dataset not used
     free(batch_dataset);
@@ -133,14 +141,12 @@ int main(int argc, char **argv) {
     const size_t network_length = 3;
     const size_t layer_lengths[] = {100, 16, 10};
 
-    neural_network *nn = get_neural_network(network_length, layer_lengths, train_dataset->inputs_length, &sigmoid);
+    neural_network *nn = get_neural_network(network_length, layer_lengths, train_dataset->inputs_length);
 
     // Parameters
-    const float learn_rate = 10.0f;
-    const size_t batch_size = 3000;
-    const int learn_amount = 50000000;
-    const int batch_amount = learn_amount / batch_size;
-    const int log_amount = 1000;  // Log once reached a number of batch
+    const float learn_rate = 0.2f;
+    const size_t batch_size = 1000;
+    const unsigned int batch_amount = 200000;
 
     char cmd[100];
     FILE *fp;
@@ -169,8 +175,8 @@ int main(int argc, char **argv) {
                 printf("Neural network loaded!\n");
             }
         } else if (cmd[0] == 't') {
-            train(nn, train_dataset, test_dataset, learn_rate, batch_amount, log_amount, batch_size);
-            printf("Training completed. Trained for %d times.\n", learn_amount);
+            train(nn, train_dataset, test_dataset, learn_rate, batch_size, batch_amount);
+            printf("Training completed. Trained for %d batches.\n", batch_amount);
         } else if (cmd[0] == 'T') {
             printf("Testing neural network...\n");
             printf("Network is %.2f%% correct!\n", test_network_percent(nn, test_dataset));

src/network.c

@@ -52,7 +52,7 @@ neural_network *alloc_neural_network(size_t network_length, const size_t *layers
     return nn;
 }
 
-neural_network *get_neural_network(size_t network_length, const size_t *layers_length, size_t inputs_length, float (*activation_function)(float, bool)) {
+neural_network *get_neural_network(size_t network_length, const size_t *layers_length, size_t inputs_length) {
     assert(layers_length);
 
     neural_network *nn = alloc_neural_network(network_length, layers_length, inputs_length);
@@ -62,8 +62,6 @@ neural_network *get_neural_network(size_t network_length, const size_t *layers_l
         // Initialise weights to -1.0f - 1.0f
         nn->weights[i] = randf(2.0f, -1.0f);
     }
-
-    nn->activation_function = activation_function;
     return nn;
 }
 
@@ -86,19 +84,19 @@ neural_network *copy_neural_network(const neural_network *nn) {
 
 void compute_network(const neural_network *restrict nn, const float *restrict inputs) {
     assert(nn && inputs);
-    cblas_sgemm(CblasColMajor, CblasNoTrans, CblasNoTrans, nn->layer_lengths[0], 1, nn->inputs_length, 1.0f, nn->weights, nn->layer_lengths[0], inputs, nn->inputs_length, 0.0f, nn->weighted_input, nn->layer_lengths[0]);
-    cblas_saxpy(nn->layer_lengths[0], 1.0f, nn->bias, 1, nn->weighted_input, 1);
-    vector_apply_activation(nn->weighted_input, nn->output, nn->layer_lengths[0], nn->activation_function, false);
+    cblas_scopy(nn->layer_lengths[0], nn->bias, 1, nn->weighted_input, 1);
+    cblas_sgemv(CblasColMajor, CblasNoTrans, nn->layer_lengths[0], nn->inputs_length, 1.0f, nn->weights, nn->layer_lengths[0], inputs, 1, 1.0f, nn->weighted_input, 1);
+    vector_apply_activation(nn->weighted_input, nn->output, nn->layer_lengths[0]);
     for (size_t i = 1; i < nn->length; i++) {
         size_t len = nn->layer_lengths[i];
         size_t prev_len = nn->layer_lengths[i - 1];
         size_t w_sum = nn->prev_weights_sums[i];
         size_t l_sum = nn->prev_lengths_sums[i];
         size_t prev_l_sum = nn->prev_lengths_sums[i - 1];
 
-        cblas_sgemm(CblasColMajor, CblasNoTrans, CblasNoTrans, len, 1, prev_len, 1.0f, &nn->weights[w_sum], len, &nn->output[prev_l_sum], prev_len, 0.0f, &nn->weighted_input[l_sum], len);
-        cblas_saxpy(len, 1.0f, &nn->bias[l_sum], 1, &nn->weighted_input[l_sum], 1);
-        vector_apply_activation(&nn->weighted_input[l_sum], &nn->output[l_sum], len, nn->activation_function, false);
+        cblas_scopy(len, &nn->bias[l_sum], 1, &nn->weighted_input[l_sum], 1);
+        cblas_sgemv(CblasColMajor, CblasNoTrans, len, prev_len, 1.0f, &nn->weights[w_sum], len, &nn->output[prev_l_sum], 1, 1.0f, &nn->weighted_input[l_sum], 1);
+        vector_apply_activation(&nn->weighted_input[l_sum], &nn->output[l_sum], len);
     }
 }
 
@@ -203,7 +201,7 @@ void layer_learn(const neural_network *nn, size_t layer_index, float learn_rate,
     size_t l_sum = nn->prev_lengths_sums[layer_index];
     size_t w_sum = nn->prev_weights_sums[layer_index];
     // f'(Z_i) in weighted_input
-    vector_apply_activation(&nn->weighted_input[l_sum], &nn->weighted_input[l_sum], len, nn->activation_function, true);
+    vector_apply_d_activation(&nn->weighted_input[l_sum], &nn->weighted_input[l_sum], len);
     if (layer_index == nn->length - 1) {
         // Error in output
         nn->output[l_sum + data_expected_index] -= 1.0f;
@@ -242,7 +240,7 @@ void layer_learn_collect_gradient(const neural_network *nn, float *restrict laye
     size_t len = nn->layer_lengths[layer_index];
     size_t l_sum = nn->prev_lengths_sums[layer_index];
     // f'(Z_i) in weighted_input
-    vector_apply_activation(&nn->weighted_input[l_sum], &nn->weighted_input[l_sum], len, nn->activation_function, true);
+    vector_apply_d_activation(&nn->weighted_input[l_sum], &nn->weighted_input[l_sum], len);
     if (layer_index == nn->length - 1) {
         // Error in output
         nn->output[l_sum + data_expected_index] -= 1.0f;
@@ -309,7 +307,7 @@ void *thread_worker(void *arg) {
     return NULL;
 }
 
-#define THREAD_COUNT 4
+#define THREAD_COUNT 5
 
 void mini_batch_gd(neural_network *nn, float learn_rate, const dataset *data_batch) {
     assert(nn && data_batch);

test/linear_algebra.cpp

@@ -16,7 +16,7 @@ TEST(LinearAlgebraTest, VectorApplyActivation) {
     a[1] = -1.2f;
     a[2] = -0.2f;
 
-    vector_apply_activation(a, b, length, sigmoid, false);
+    vector_apply_activation(a, b, length);
 
     for (size_t i = 0; i < length; i++) {
         ASSERT_FLOAT_EQ(b[i], sigmoid(a[i], false));

test/network.cpp

@@ -25,12 +25,11 @@ TEST(NetworkTest, GetNeuralNetwork) {
     const size_t layer_lengths[] = {2, 3, 4};
     const size_t inputs_length = 2;
 
-    neural_network *nn = get_neural_network(network_length, layer_lengths, inputs_length, &sigmoid);
+    neural_network *nn = get_neural_network(network_length, layer_lengths, inputs_length);
 
     ASSERT_NE(nn, nullptr);
     EXPECT_EQ(nn->length, network_length);
     EXPECT_EQ(nn->inputs_length, inputs_length);
-    EXPECT_EQ(nn->activation_function, &sigmoid);
     ASSERT_NE(nn->layer_lengths, nullptr);
     ASSERT_NE(nn->prev_lengths_sums, nullptr);
     ASSERT_NE(nn->prev_weights_sums, nullptr);
@@ -61,7 +60,7 @@ TEST(NetworkTest, FreeDataset) {
     size_t layer_length = 1;
     size_t *layer_lengths = (size_t *)malloc(sizeof(size_t) * layer_length);
     layer_lengths[0] = 2;
-    neural_network *nn = get_neural_network(layer_length, layer_lengths, 2, nullptr);
+    neural_network *nn = get_neural_network(layer_length, layer_lengths, 2);
 
     free(nn);
     // No crash
@@ -73,7 +72,7 @@ TEST(NetworkTest, ComputeNetwork) {
     size_t inputs_length = 1;
     size_t *layer_lengths = (size_t *)malloc(sizeof(size_t) * layer_length);
     layer_lengths[0] = 1;
-    neural_network *nn = get_neural_network(layer_length, layer_lengths, inputs_length, &sigmoid);
+    neural_network *nn = get_neural_network(layer_length, layer_lengths, inputs_length);
 
     float *inputs = (float *)malloc(sizeof(float) * inputs_length);
     inputs[0] = 0.2f;
@@ -95,7 +94,7 @@ TEST(NetworkTest, Softmax) {
     size_t inputs_length = 1;
     size_t *layer_lengths = (size_t *)malloc(sizeof(size_t) * layer_length);
     layer_lengths[0] = 3;
-    neural_network *nn = get_neural_network(layer_length, layer_lengths, inputs_length, &sigmoid);
+    neural_network *nn = get_neural_network(layer_length, layer_lengths, inputs_length);
 
     nn->output[0] = 0.2f;
     nn->output[1] = 0.3f;
@@ -117,7 +116,7 @@ TEST(NetworkTest, StochasticGDSingleLayer) {
     size_t layer_length = 1;
     size_t *layer_lengths = (size_t *)malloc(sizeof(size_t) * layer_length);
     layer_lengths[0] = 2;
-    neural_network *nn = get_neural_network(layer_length, layer_lengths, test_dataset->inputs_length, &sigmoid);
+    neural_network *nn = get_neural_network(layer_length, layer_lengths, test_dataset->inputs_length);
 
     for (size_t i = 0; i < 50000; i++) {
         for (size_t j = 0; j < test_dataset->length; j++) {
@@ -145,7 +144,7 @@ TEST(NetworkTest, StochasticGDTests) {
     size_t *layer_lengths = (size_t *)malloc(sizeof(size_t) * layer_length);
     layer_lengths[0] = 4;
     layer_lengths[1] = 2;
-    neural_network *nn = get_neural_network(layer_length, layer_lengths, test_dataset->inputs_length, &sigmoid);
+    neural_network *nn = get_neural_network(layer_length, layer_lengths, test_dataset->inputs_length);
 
     for (size_t i = 0; i < 500000; i++) {
         for (size_t j = 0; j < test_dataset->length; j++) {
@@ -167,7 +166,7 @@ TEST(NetworkTest, StochasticGDTests) {
     layer_length = 1;
     layer_lengths = (size_t *)malloc(sizeof(size_t) * layer_length);
     layer_lengths[0] = 2;
-    nn = get_neural_network(layer_length, layer_lengths, test_dataset->inputs_length, &sigmoid);
+    nn = get_neural_network(layer_length, layer_lengths, test_dataset->inputs_length);
 
     for (size_t i = 0; i < 50000; i++) {
         for (size_t j = 0; j < test_dataset->length; j++) {
@@ -195,7 +194,7 @@ TEST(NetworkTest, MiniBatchGDTests) {
     size_t *layer_lengths = (size_t *)malloc(sizeof(size_t) * layer_length);
     layer_lengths[0] = 4;
     layer_lengths[1] = 2;
-    neural_network *nn = get_neural_network(layer_length, layer_lengths, test_dataset->inputs_length, &sigmoid);
+    neural_network *nn = get_neural_network(layer_length, layer_lengths, test_dataset->inputs_length);
 
     for (size_t i = 0; i < 2000000; i++) {
         dataset *batch_dataset = get_random_dataset_sample(test_dataset, test_dataset->length);
@@ -216,7 +215,7 @@ TEST(NetworkTest, MiniBatchGDTests) {
     layer_length = 1;
     layer_lengths = (size_t *)malloc(sizeof(size_t) * layer_length);
     layer_lengths[0] = 2;
-    nn = get_neural_network(layer_length, layer_lengths, test_dataset->inputs_length, &sigmoid);
+    nn = get_neural_network(layer_length, layer_lengths, test_dataset->inputs_length);
 
     for (size_t i = 0; i < 100000; i++) {
         dataset *batch_dataset = get_random_dataset_sample(test_dataset, test_dataset->length);