Commit aa962848 authored by Petr Kubeš's avatar Petr Kubeš

add regularizations and prepare UI for training input

parent 2846327c
This source diff could not be displayed because it is too large. You can view the blob instead.
......@@ -39,15 +39,16 @@
</div>
<div class="row">
<label for="regularization-input" class="col-8">Regularization:</label>
<select class="form-control form-control-sm col-4" id="regularization-input">
<select class="form-control form-control-sm col-4" id="regularization-type-input">
<option value="none">None</option>
<option value="L1">L1</option>
<option value="L2">L2</option>
</select>
</div>
<div class="row">
<label for="regularization-input" class="col-8">Regularization rate:</label>
<input class="form-control form-control-sm col-4" id="regularization-rate-input" type="number" name="rate" min="0" max="10"
value="1">
step="0.1" value="0.1">
</div>
</div>
<div class="col-md-3">
......@@ -64,13 +65,55 @@
</div>
</div>
<hr>
<div class="row">
<div class="col-4">
<h5>Training set input</h5>
<textarea class="form-control" id="exampleFormControlTextarea1" rows="3"></textarea>
<button type="button" class="btn btn-outline-primary btn-sm mt-3">Apply training data</button>
</div>
<div class="col-8">
<h5>Training set</h5>
<div id="training-set-output"></div>
<table class="table table-sm table-bordered table-hover">
<thead class="thead-dark">
<tr>
<th scope="col">#</th>
<th scope="col">Neuron 1</th>
<th scope="col">Neuron 1</th>
<th scope="col">Neuron 1</th>
</tr>
</thead>
<tbody>
<tr>
<th scope="row">1</th>
<td>Mark</td>
<td>Otto</td>
<td>@mdo</td>
</tr>
<tr>
<th scope="row">2</th>
<td>Jacob</td>
<td>Thornton</td>
<td>@fat</td>
</tr>
<tr>
<th scope="row">3</th>
<td>Larry</td>
<td>the Bird</td>
<td>@twitter</td>
</tr>
</tbody>
</table>
</div>
</div>
<hr>
<div class="row mb-3">
<div class="col-md-auto">
<button type="button" class="btn btn-primary mr-2" onclick="train(true)">Train</button>
<button type="button" class="btn btn-secondary" onclick="train(false)">Step</button>
<button type="button" class="btn btn-danger" onclick="reset()">Reset</button>
</div>
<div class="col-md">
<div class="col-md-auto">
Cost:
<b id="cost"></b>
</div>
......
import { Visualizer } from './Visualizer';
import { NeuralCore } from './neuralNetwork/NeuralCore';
import { Regularizations } from './neuralNetwork/HelperObjects';
(window as any).slide = (i: number, value: number) => {
input[i] = value;
......@@ -33,6 +34,21 @@ import { NeuralCore } from './neuralNetwork/NeuralCore';
let iters = multipleIters ? Number.parseInt(itersInput.value) : 1;
neuralCore.setRate(Number.parseFloat(rateInput.value));
// Regularization
switch (regTypeInput.value) {
case "L1":
neuralCore.setRegularizationType(Regularizations.L1);
break;
case "L2":
neuralCore.setRegularizationType(Regularizations.L2);
break;
case "none":
neuralCore.setRegularizationType(Regularizations.NONE);
break;
}
neuralCore.setRegularizationRate(Number.parseFloat(regRateInput.value));
for (let i = 0; i < iters; i++) {
neuralCore.train();
}
......@@ -49,7 +65,6 @@ import { NeuralCore } from './neuralNetwork/NeuralCore';
visualizer.draw(neuralCore.getNeurons(), neuralCore.getConnections());
}
window.onload = () => {
main();
};
......@@ -69,6 +84,8 @@ let iter: HTMLElement;
let rateInput: HTMLInputElement;
let itersInput: HTMLInputElement;
let regTypeInput: HTMLInputElement;
let regRateInput: HTMLInputElement;
const main = () => {
const content: HTMLCanvasElement = document.getElementById('content') as HTMLCanvasElement;
......@@ -78,6 +95,8 @@ const main = () => {
cost = document.getElementById('cost');
rateInput = document.getElementById('rate-input') as HTMLInputElement;
itersInput = document.getElementById('iters-input') as HTMLInputElement;
regTypeInput = document.getElementById('regularization-type-input') as HTMLInputElement;
regRateInput = document.getElementById('regularization-rate-input') as HTMLInputElement;
visualizer = new Visualizer(content);
......
export interface Activation {
der(x: number): number;
output(x: number): number;
}
export class Activations {
public static SIGMOID: Activation = {
output: (x: number): number => 1 / (1 + Math.exp(-x)),
der: (x: number): number => {
let output = Activations.SIGMOID.output(x);
return output * (1 - output);
}
};
}
export class TrainSample {
public input: number[];
public output: number[];
constructor(input: number[], output: number[]) {
this.input = input;
this.output = output;
}
}
\ No newline at end of file
import { Connection } from "./Connection";
export interface Activation {
der(x: number): number;
output(x: number): number;
}
export const SIGMOID: Activation = {
output: (x: number): number => 1 / (1 + Math.exp(-x)),
der: (x: number): number => {
let output = SIGMOID.output(x);
return output * (1 - output);
}
};
export enum Regularizations {
L1,
L2,
NONE,
}
export const L1Reg = {
cost: (connections: Connection[][]): number => {
return connections.reduce(
(prev, connLayer: Connection[]) => {
return prev + Math.abs(connLayer.reduce((acc, conn) => acc + conn.getWeight(), 0))
}, 0) * (1 / getNumberOfConnections(connections));
},
der: (weight: number): number => {
return (weight > 0) ? 1 : -1;
}
}
export const L2Reg = {
cost: (connections: Connection[][]): number => {
return 1 / 2 * connections.reduce(
(prev, connLayer: Connection[]) => {
return prev + connLayer.reduce((acc, conn) => acc + conn.getWeight(), 0) ** 2
}, 0) * (1 / getNumberOfConnections(connections));
},
der: (currWeight: number, connCount: number): number => {
return currWeight * (1 / connCount);
}
}
export class TrainSample {
public input: number[];
public output: number[];
constructor(input: number[], output: number[]) {
this.input = input;
this.output = output;
}
}
export const getNumberOfConnections = (connections: Connection[][]): number => {
return connections.reduce((acc, conn) => acc + conn.length, 0);
}
\ No newline at end of file
import { Neuron } from "./Neuron";
import { Connection } from "./Connection";
import { TrainSample, Activations } from "./HelperClasses";
import { TrainSample, SIGMOID, Regularizations, L2Reg, getNumberOfConnections, L1Reg } from "./HelperObjects";
export class NeuralCore {
private inputSize: number;
......@@ -13,6 +13,7 @@ export class NeuralCore {
private rate = 1;
private lambda = 0.001;
private regType: Regularizations = Regularizations.NONE;
private biasNeuron = new Neuron('bias', true);
private neurons: Neuron[][] = [];
......@@ -97,11 +98,22 @@ export class NeuralCore {
}, 0);
}, 0);
// Regularization
let regCost = 0;
switch (this.regType) {
case Regularizations.L1:
regCost = L1Reg.cost(this.connections);
break;
case Regularizations.L2:
regCost = L2Reg.cost(this.connections);
break;
case Regularizations.NONE:
regCost = 0;
break;
}
return 1 / 2 * costSum * (1 / this.trainSamples.length) +
1 / 2 * this.lambda * this.connections.reduce( // Regularization
(prev, connLayer: Connection[]) => {
return prev + connLayer.reduce((acc, conn) => acc + conn.getWeight(), 0) ** 2
}, 0) * (1 / this.getNumberOfConnections());
this.lambda * regCost;
}
public train() {
......@@ -111,7 +123,7 @@ export class NeuralCore {
// Calculate sigmas of the last layer
this.neurons[this.layerCnt - 1].forEach((neuron, idx) => {
const newSigma =
(sample.output[idx] - neuron.getActivation()) * Activations.SIGMOID.der(neuron.getActivation());
(sample.output[idx] - neuron.getActivation()) * SIGMOID.der(neuron.getActivation());
neuron.setSigma(newSigma);
});
......@@ -122,7 +134,7 @@ export class NeuralCore {
const newSigma =
neuron.getOutputs().reduce((acc, connection) => {
return acc + connection.getOutputNeuron().getSigma() * connection.getWeight();
}, 0) * Activations.SIGMOID.der(neuron.getActivation());
}, 0) * SIGMOID.der(neuron.getActivation());
neuron.setSigma(newSigma);
});
}
......@@ -130,11 +142,25 @@ export class NeuralCore {
// Accumulate all weight updates
this.connections.forEach((connLayer) => {
connLayer.forEach((connection) => {
let regDer = 0;
switch (this.regType) {
case Regularizations.L1:
regDer = L1Reg.der(connection.getWeight());
break;
case Regularizations.L2:
regDer = L2Reg.der(connection.getWeight(), getNumberOfConnections(this.connections));
break;
case Regularizations.NONE:
regDer = 0;
break;
}
const weightChange =
connection.getOutputNeuron().getSigma() *
connection.getInputNeuron().getActivation() *
this.rate -
this.lambda * connection.getWeight() * (1 / this.getNumberOfConnections()); // Regularization
this.lambda * regDer; // Regularization
connection.addSampleWeightChange(weightChange);
});
......@@ -314,10 +340,6 @@ export class NeuralCore {
}
}
private getNumberOfConnections(): number {
return this.connections.reduce((acc, conn) => acc + conn.length, 0);
}
public getNeurons() {
return this.neurons;
}
......@@ -349,4 +371,12 @@ export class NeuralCore {
public getIteration() {
return this.iterCnt;
}
public setRegularizationType(regType: Regularizations) {
this.regType = regType;
}
public setRegularizationRate(rate: number) {
this.lambda = rate;
}
}
import { Connection } from "./Connection";
import { Activations } from "./HelperClasses";
import { SIGMOID } from "./HelperObjects";
export class Neuron {
private name: string;
......@@ -92,7 +92,7 @@ export class Neuron {
public calculateActivation(): number {
if (!this.isInput && !this.isCalculated && !this.isBias) {
this.activation = Activations.SIGMOID.output(this.inputs.reduce((acc, currConn) => acc + currConn.calculateValue(), 0));
this.activation = SIGMOID.output(this.inputs.reduce((acc, currConn) => acc + currConn.calculateValue(), 0));
this.isCalculated = true;
}
return this.getActivation();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment