#include #include #include #include #include "Neural_network.h" void Neural_network::allocate_matrices () { // Time to allocate the entire neural_net structure // Activation matrices hidden1_act = new double [num_hidden1]; hidden2_act = new double [num_hidden2]; output_act = new double [num_outputs]; // Weight matrices input_weights = new double [num_hidden1 * num_inputs]; hidden1_weights = new double [num_hidden2 * num_hidden1]; hidden2_weights = new double [num_outputs * num_hidden2]; // Learning rate matrices for each weight's learning rate. // Needed for delta-bar-delta algorithm input_learning_rate = new double [num_hidden1 * num_inputs]; hidden1_learning_rate = new double [num_hidden2 * num_hidden1]; hidden2_learning_rate = new double [num_outputs * num_hidden2]; // Learning rate deltas for each weight's learning rate. // Needed for delta-bar-delta algorithm. input_learning_delta = new double [num_hidden1 * num_inputs]; hidden1_learning_delta = new double [num_hidden2 * num_hidden1]; hidden2_learning_delta = new double [num_outputs * num_hidden2]; // Delta bar matrices for each weight's delta bar. // Needed for delta-bar-delta algorithm. input_learning_delta_bar = new double [num_hidden1 * num_inputs]; hidden1_learning_delta_bar = new double [num_hidden2 * num_hidden1]; hidden2_learning_delta_bar = new double [num_outputs * num_hidden2]; // Weight delta matrices for each weights delta. // Needed for BackPropagation algorithm. input_weights_sum_delta = new double [num_hidden1 * num_inputs]; hidden1_weights_sum_delta = new double [num_hidden2 * num_hidden1]; hidden2_weights_sum_delta = new double [num_outputs * num_hidden2]; // Sum of delta * weight matrices for each weight. // Needed for BackPropagation algorithm. hidden1_sum_delta_weight = new double [num_hidden2 * num_hidden1]; hidden2_sum_delta_weight = new double [num_outputs * num_hidden2]; /* Done neural net allocation */ } void Neural_network::initialize_matrices (double range) { int x,y; double rand_max; training_examples = 0; examples_since_update = 0; rand_max = RAND_MAX; /* Initialize all weights from -range to +range randomly */ for (x = 0; x < num_hidden1; ++x) { hidden1_sum_delta_weight [x] = 0.0; for (y = 0; y < num_inputs; ++y) { input_weights [x * num_inputs + y] = rand () / rand_max * range; if ( rand () < (RAND_MAX / 2) ) input_weights [x * num_inputs + y] *= -1.0; input_weights_sum_delta [x * num_inputs + y] = 0.0; input_learning_rate [x * num_inputs + y] = learning_rate; input_learning_delta [x * num_inputs + y] = 0.0; input_learning_delta_bar [x * num_inputs + y] = 0.0; } } for (x = 0; x < num_hidden2; ++x) { hidden2_sum_delta_weight [x] = 0.0; for (y = 0; y < num_hidden1; ++y) { hidden1_weights [x * num_hidden1 + y] = rand () / rand_max * range; if ( rand () < (RAND_MAX / 2) ) hidden1_weights [x * num_hidden1 + y] *= -1.0; hidden1_weights_sum_delta [x * num_hidden1 + y] = 0.0; hidden1_learning_rate [x * num_hidden1 + y] = learning_rate; hidden1_learning_delta [x * num_hidden1 + y] = 0.0; hidden1_learning_delta_bar [x * num_hidden1 + y] = 0.0; } } for (x = 0; x < num_outputs; ++x) { for (y = 0; y < num_hidden2; ++y) { hidden2_weights [x * num_hidden2 + y] = rand () / rand_max * range; if ( rand () < (RAND_MAX / 2) ) hidden2_weights [x * num_hidden2 + y] *= -1.0; hidden2_weights_sum_delta [x * num_hidden2 + y] = 0.0; hidden2_learning_rate [x * num_hidden2 + y] = learning_rate; hidden2_learning_delta [x * num_hidden2 + y] = 0.0; hidden2_learning_delta_bar [x * num_hidden2 + y] = 0.0; } } } void Neural_network::deallocate_matrices () { // Time to destroy the entire neural_net structure // Activation matrices delete hidden1_act; delete hidden2_act; delete output_act; // Weight matrices delete input_weights; delete hidden1_weights; delete hidden2_weights; // Learning rate matrices for each weight's learning rate. // Needed for delta-bar-delta algorithm delete input_learning_rate; delete hidden1_learning_rate; delete hidden2_learning_rate; // Learning rate deltas for each weight's learning rate. // Needed for delta-bar-delta algorithm. delete input_learning_delta; delete hidden1_learning_delta; delete hidden2_learning_delta; // Delta bar matrices for each weight's delta bar. // Needed for delta-bar-delta algorithm. delete input_learning_delta_bar; delete hidden1_learning_delta_bar; delete hidden2_learning_delta_bar; // Weight delta matrices for each weights delta. // Needed for BackPropagation algorithm. delete input_weights_sum_delta; delete hidden1_weights_sum_delta; delete hidden2_weights_sum_delta; // Sum of delta * weight matrices for each weight. // Needed for BackPropagation algorithm. delete hidden1_sum_delta_weight; delete hidden2_sum_delta_weight; /* Done neural net deallocation */ } Neural_network::Neural_network (int number_inputs, int number_hidden1, int number_hidden2, int number_outputs, double t_epsilon, double t_skip_epsilon, double t_learning_rate, double t_theta, double t_phi, double t_K, double range) : num_inputs (number_inputs), num_hidden1 (number_hidden1), num_hidden2 (number_hidden2), num_outputs (number_outputs), epsilon (t_epsilon), skip_epsilon (t_skip_epsilon), learning_rate (t_learning_rate), theta (t_theta), phi (t_phi), K (t_K), training_examples (0), examples_since_update (0) { allocate_matrices (); initialize_matrices (range); } Neural_network::Neural_network (String& filename, int& file_error, double t_epsilon, double t_skip_epsilon, double t_learning_rate, double t_theta, double t_phi, double t_K) : epsilon (t_epsilon), skip_epsilon (t_skip_epsilon), learning_rate (t_learning_rate), theta (t_theta), phi (t_phi), K (t_K), examples_since_update (0), hidden1_act (0), hidden2_act (0), output_act (0), input_weights (0), hidden1_weights (0), hidden2_weights (0), input_learning_rate (0), hidden1_learning_rate (0), hidden2_learning_rate (0), input_learning_delta (0), hidden1_learning_delta (0), hidden2_learning_delta (0), input_learning_delta_bar (0), hidden1_learning_delta_bar (0), hidden2_learning_delta_bar (0), input_weights_sum_delta (0), hidden1_weights_sum_delta (0), hidden2_weights_sum_delta (0), hidden1_sum_delta_weight (0), hidden2_sum_delta_weight (0) { file_error = read_weights (filename); } int Neural_network::read_weights (String& filename) { ifstream fp; int x; long iter; fp.open (filename); if ( fp.fail() != 0 ) { cout << "Could not read weights from file " << filename << "\n"; return (-1); } /* First read in how many iterations have been performed so far */ fp >> iter; cout << "Iterations = " << iter << "\n"; /* Next read in how many input nodes, hidden1 nodes, hidden2 nodes */ /* and output nodes. */ fp >> num_inputs >> num_hidden1 >> num_hidden2 >> num_outputs; // Deallocate previous matrices deallocate_matrices (); /* Allocate new matrices with new size */ allocate_matrices (); // Initialize all matrices and variables initialize_matrices (1.0); training_examples = iter; /* Read input->hidden1 weights from file. */ for (x = 0; x < num_inputs * num_hidden1; ++x) { fp >> input_weights [x]; } /* Read hidden1->hidden2 weights from file. */ for (x = 0; x < num_hidden1 * num_hidden2; ++x) { fp >> hidden1_weights [x]; } /* Read hidden2->output weights from file. */ for (x = 0; x < (num_hidden2 * num_outputs); ++x) { fp >> hidden2_weights [x]; } fp.close (); /* Now all the weights have been loaded */ return (0); } int Neural_network::save_weights (String& filename) { ofstream fp; int x; fp.open (filename); if ( fp.fail() != 0 ) { cout << "Could not save weights to file " << filename << "\n"; return (-1); } /* First write out how many iterations have been performed so far */ fp << training_examples << "\n"; /* Next write out how many input nodes, hidden1 nodes, hidden2 nodes */ /* and output nodes. */ fp << num_inputs << " " << num_hidden1 << " " << num_hidden2 << " "; fp << num_outputs << "\n"; fp.precision (6); /* Write input->hidden1 weights to output. */ for (x = 0; x < num_inputs * num_hidden1; ++x) { fp.width (10); fp << input_weights [x] << " "; if ( (x % 5) == 4 ) fp << "\n"; } fp << "\n\n"; /* Write hidden1->hidden2 weights to output. */ for (x = 0; x < num_hidden1 * num_hidden2; ++x) { fp.width (10); fp << hidden1_weights [x] << " "; if ( (x % 5) == 4 ) fp << "\n"; } fp << "\n\n"; /* Write hidden2->output weights to output. */ for (x = 0; x < (num_hidden2 * num_outputs); ++x) { fp.width (10); fp << hidden2_weights [x] << " "; if ( (x % 5) == 4 ) fp << "\n"; } fp << "\n\n"; fp.close (); cout << "Closed file\n"; /* Now all the weights have been saved */ return (0); } void Neural_network::set_size_parameters (int number_inputs, int number_hidden1, int number_hidden2, int number_outputs, double range) { double *new_input_weights,*new_hidden1_weights,*new_hidden2_weights; double rand_max; int x; rand_max = RAND_MAX; // Allocate new weight matrices with new size new_input_weights = new double [number_hidden1 * number_inputs]; new_hidden1_weights = new double [number_hidden2 * number_hidden1]; new_hidden2_weights = new double [number_outputs * number_hidden2]; // Copy over all weights // Input weights for (x = 0; x < number_hidden1 * number_inputs; ++x) { // IF the new size is larger than the old size, THEN make new connections // a random weight between +-range. if ( x >= (num_hidden1 * num_inputs) ) { new_input_weights [x] = rand () / rand_max * range; if ( rand () < (RAND_MAX / 2) ) new_input_weights [x] *= -1.0; } else new_input_weights [x] = input_weights [x]; } // Hidden1 weights for (x = 0; x < number_hidden2 * number_hidden1; ++x) { // IF the new size is larger than the old size, THEN make new connections // a random weight between +-range. if ( x >= (num_hidden2 * num_hidden1) ) { new_hidden1_weights [x] = rand () / rand_max * range; if ( rand () < (RAND_MAX / 2) ) new_hidden1_weights [x] *= -1.0; } else new_hidden1_weights [x] = hidden1_weights [x]; } // Hidden2 weights for (x = 0; x < number_outputs * number_hidden2; ++x) { // IF the new size is larger than the old size, THEN make new connections // a random weight between +-range. if ( x >= (num_outputs * num_hidden2) ) { new_hidden2_weights [x] = rand () / rand_max * range; if ( rand () < (RAND_MAX / 2) ) new_hidden2_weights [x] *= -1.0; } else new_hidden2_weights [x] = hidden2_weights [x]; } // All weights have been copied. // Change size paramters num_inputs = number_inputs; num_hidden1 = number_hidden1; num_hidden2 = number_hidden2; num_outputs = number_outputs; // Deallocate all matrices deallocate_matrices (); // Allocate new nerual network matrices with the correct size and initialize allocate_matrices (); initialize_matrices (1.0); // Now deallocate new randomly initialized weight matrices and assign them // to the new weight matrices that have the correct weight values. delete input_weights; delete hidden1_weights; delete hidden2_weights; input_weights = new_input_weights; hidden1_weights = new_hidden1_weights; hidden2_weights = new_hidden2_weights; } void Neural_network::back_propagation (double input [], double desired_output [], int& done) { int x,y; int size; double delta,sum_delta,*weight,*p_sum_delta,*p_learning_delta; /* First check if training complete. */ for (x = num_outputs - 1; x >= 0; --x) { if ( fabs (desired_output [x] - output_act [x]) > epsilon ) { done = 0; } } /* Go backward through list for speed */ size = num_hidden2; /* First calculate deltas of weights from output to hidden layer 2. */ for (x = num_outputs - 1; x >= 0; --x) { weight = &hidden2_weights [x * size]; p_sum_delta = &hidden2_weights_sum_delta [x * size]; p_learning_delta = &hidden2_learning_delta [x * size]; for (y = num_hidden2 - 1; y >= 0; --y) { /* Formula delta = (desired - actual) * derivative derivative = S(1 - S) Also calculate sum of deltas * weight for next layer. */ delta = (desired_output [x] - output_act [x]) * SLOPE * output_act [x] * (1.0 - output_act [x]); hidden2_sum_delta_weight [y] += delta * weight [y]; p_learning_delta [y] += delta; /* Now multiply by activation and sum in weights sum delta */ p_sum_delta [y] += delta * hidden2_act [y]; } } /* Next calculate deltas of weights between hidden layer2 and hidden layer 1 */ size = num_hidden1; for (x = num_hidden2 - 1; x >= 0; --x) { sum_delta = hidden2_sum_delta_weight [x]; weight = &hidden1_weights [x * size]; p_sum_delta = &hidden1_weights_sum_delta [x * size]; p_learning_delta = &hidden1_learning_delta [x * size]; for (y = num_hidden1 - 1; y >= 0; --y) { /* Formula delta = SUM (previous deltas*weight) * derivative previous deltas already muliplied by weight. derivative = S(1 - S) Also calculate sum of deltas * weight to save from doing it for next layer. */ delta = sum_delta * hidden2_act [x] * (1.0 - hidden2_act [x]) * SLOPE; hidden1_sum_delta_weight [y] += delta * weight [y]; p_learning_delta [y] += delta; /* Now multiply by activation and sum in weights_sum_delta */ p_sum_delta [y] += delta * hidden1_act [y]; } hidden2_sum_delta_weight [x] = 0.0; } /* Finally calculate deltas of weights between hidden layer 1 and input layer */ size = num_inputs; for (x = num_hidden1 - 1; x >= 0; --x) { sum_delta = hidden1_sum_delta_weight [x]; p_sum_delta = &input_weights_sum_delta [x * size]; p_learning_delta = &input_learning_delta [x * size]; for (y = num_inputs - 1; y >= 0; --y) { /* Formula delta = SUM (previous deltas*weight) * derivative * activation of input previous deltas already muliplied by weight derivative = S(1 - S) */ delta = sum_delta * hidden1_act [x] * (1.0 - hidden1_act [x]) * SLOPE; p_learning_delta [y] += delta; p_sum_delta [y] += (delta * input [y]); } hidden1_sum_delta_weight [x] = 0.0; } /* Now all deltas have been calculated and added into their appropriate neuron connection. */ ++examples_since_update; } double Neural_network::calc_forward (double input [], double desired_output [], int& num_wrong, int& skip, int print_it, int& actual_printed) { int x,y,wrong; int size; double *weight,error,abs_error; skip = 1; wrong = 0; error = 0.0; /* Go backward for faster processing */ /* Calculate hidden layer 1's activation */ size = num_inputs; for (x = num_hidden1 - 1; x >= 0; --x) { hidden1_act [x] = 0.0; weight = &input_weights [x * size]; for (y = num_inputs - 1; y >= 0; --y) { hidden1_act [x] += (input [y] * weight [y]); } hidden1_act [x] = S(hidden1_act [x]); } /* Calculate hidden layer 2's activation */ size = num_hidden1; for (x = num_hidden2 - 1; x >= 0; --x) { hidden2_act [x] = 0.0; weight = &hidden1_weights [x * size]; for (y = num_hidden1 - 1; y >= 0; --y) { hidden2_act [x] += (hidden1_act [y] * weight [y]); } hidden2_act [x] = S(hidden2_act [x]); } /* Calculate output layer's activation */ size = num_hidden2; for (x = num_outputs - 1; x >= 0; --x) { output_act [x] = 0.0; weight = &hidden2_weights [x * size]; for (y = num_hidden2 - 1; y >= 0; --y) { output_act [x] += hidden2_act [y] * weight [y]; } output_act [x] = S(output_act [x]); abs_error = fabs (output_act [x] - desired_output [x]); error += abs_error; if ( abs_error > epsilon ) wrong = 1; if ( abs_error > skip_epsilon ) skip = 0; } if ( wrong ) ++num_wrong; cout.precision (3); if ( print_it == 2 ) { for (x = 0; x < num_outputs; ++x) { cout.width (6); cout << output_act [x] << " "; } ++actual_printed; } else if ( print_it && wrong ) { for (x = 0; x < num_outputs; ++x) { cout.width (6); cout << fabs (desired_output [x] - output_act [x]) << " "; } ++actual_printed; } return (error); } void Neural_network::update_weights () { int x,y; int size; double rate,*p_ldelta,*p_ldelta_bar,*weight,*p_lrate,*p_sum_delta; // Check to see if any changes have been calculated. if ( examples_since_update == 0 ) { return; } /* Go backward for slightly faster processing */ /* First add deltas of weights from output to hidden layer 2. */ size = num_hidden2; for (x = num_outputs - 1; x >= 0; --x) { p_ldelta = &hidden2_learning_delta [x * size]; p_ldelta_bar = &hidden2_learning_delta_bar [x * size]; weight = &hidden2_weights [x * size]; p_lrate = &hidden2_learning_rate [x * size]; p_sum_delta = &hidden2_weights_sum_delta [x * size]; for (y = num_hidden2 - 1; y >= 0; --y) { rate = p_ldelta [y] * p_ldelta_bar [y]; if ( rate < 0.0 ) { p_lrate [y] -= (phi * p_lrate [y]); } else if ( rate > 0.0 ) { p_lrate [y] += K; } weight [y] += (p_lrate [y] * p_sum_delta [y]); p_sum_delta [y] = 0.0; p_ldelta_bar [y] *= theta; p_ldelta_bar [y] += ((1.0 - theta) * p_ldelta [y]); p_ldelta [y] = 0.0; } } /* Next add deltas of weights between hidden layer2 and hidden layer 1 */ size = num_hidden1; for (x = num_hidden2 - 1; x >= 0; --x) { p_ldelta = &hidden1_learning_delta [x * size]; p_ldelta_bar = &hidden1_learning_delta_bar [x * size]; weight = &hidden1_weights [x * size]; p_lrate = &hidden1_learning_rate [x * size]; p_sum_delta = &hidden1_weights_sum_delta [x * size]; for (y = num_hidden1 - 1; y >= 0; --y) { rate = p_ldelta [y] * p_ldelta_bar [y]; if ( rate < 0.0 ) { p_lrate [y] -= (phi * p_lrate [y]); } else if ( rate > 0.0 ) { p_lrate [y] += K; } weight [y] += (p_lrate [y] * p_sum_delta [y]); p_sum_delta [y] = 0.0; p_ldelta_bar [y] *= theta; p_ldelta_bar [y] += ((1.0 - theta) * p_ldelta [y]); p_ldelta [y] = 0.0; } } /* Finally add deltas of weights between hidden layer 1 and input layer */ size = num_inputs; for (x = num_hidden1 - 1; x >= 0; --x) { p_ldelta = &input_learning_delta [x * size]; p_ldelta_bar = &input_learning_delta_bar [x * size]; weight = &input_weights [x * size]; p_lrate = &input_learning_rate [x * size]; p_sum_delta = &input_weights_sum_delta [x * size]; for (y = num_inputs - 1; y >= 0; --y) { rate = p_ldelta [y] * p_ldelta_bar [y]; if ( rate < 0.0 ) { p_lrate [y] -= (phi * p_lrate [y]); } else if ( rate > 0.0 ) { p_lrate [y] += K; } weight [y] += (p_lrate [y] * p_sum_delta [y]); p_sum_delta [y] = 0.0; p_ldelta_bar [y] *= theta; p_ldelta_bar [y] += ((1.0 - theta) * p_ldelta [y]); p_ldelta [y] = 0.0; } } /* Now all deltas have been added into their appropriate neuron connection. */ training_examples += examples_since_update; examples_since_update = 0; } int Neural_network::calc_forward_test (double input [], double desired_output [], int print_it, double correct_eps, double good_eps) { int x,y,wrong,good; wrong = 0; good = 0; /* Go backward for faster processing */ /* Calculate hidden layer 1's activation */ for (x = num_hidden1 - 1; x >= 0; --x) { hidden1_act [x] = 0.0; for (y = num_inputs - 1; y >= 0; --y) { hidden1_act [x] += (input [y] * input_weights [x * num_inputs + y]); } hidden1_act [x] = S(hidden1_act [x]); } /* Calculate hidden layer 2's activation */ for (x = num_hidden2 - 1; x >= 0; --x) { hidden2_act [x] = 0.0; for (y = num_hidden1 - 1; y >= 0; --y) { hidden2_act [x] += (hidden1_act [y] * hidden1_weights [x*num_hidden1 + y]); } hidden2_act [x] = S(hidden2_act [x]); } /* Calculate output layer's activation */ for (x = num_outputs - 1; x >= 0; --x) { output_act [x] = 0.0; for (y = num_hidden2 - 1; y >= 0; --y) { output_act [x] += hidden2_act [y] * hidden2_weights [x * num_hidden2 + y]; } output_act [x] = S(output_act [x]); if ( fabs (output_act [x] - desired_output [x]) > good_eps ) wrong = 1; else if ( fabs (output_act [x] - desired_output [x]) > correct_eps ) good = 1; } cout.precision(3); if ( print_it ) { for (x = 0; x < num_outputs; ++x) { cout.width (6); cout << output_act [x] << " "; } } if ( wrong ) return (WRONG); else if ( good ) return (GOOD); else return (CORRECT); } void Neural_network::kick_weights (double range) { int x; double rand_max; double variation; rand_max = RAND_MAX; /* Add from -range to +range to all weights randomly */ for (x = 0; x < (num_hidden1 * num_inputs); ++x) { variation = rand () / rand_max * range; if ( rand () < (RAND_MAX / 2) ) variation *= -1.0; input_weights [x] += variation; } for (x = 0; x < (num_hidden2 * num_hidden1); ++x) { variation = rand () / rand_max * range; if ( rand () < (RAND_MAX / 2) ) variation *= -1.0; hidden1_weights [x] += variation; } for (x = 0; x < (num_outputs * num_hidden2); ++x) { variation = rand () / rand_max * range; if ( rand () < (RAND_MAX / 2) ) variation *= -1.0; hidden2_weights [x] += variation; } }