Question about Perceptron activation threshold - neural-network

Question

I have a Perceptron written in Javascript that works fine, code below. My question is about the threshold in the activation function. Other code I have seen has something like if (sum > 0) {return 1} else {return 0}. My perceptron only works with if (sum > 1) {return 1} else {return 0}. Why is that? Full code below.
(function () {
"use strict";
function Perceptron(numInputs=2) {
let weights = Array.from({length: numInputs}, () => 2 * Math.random() - 1); // [-1, 1)
this.learningRate = 0.5;
this.train = function (inputs, goal) {
// feed forward
let guess = this.predict(inputs);
// back propagation
let error = goal - guess;
if (error !== 0) {
for (let i = 0; i < weights.length; i += 1) {
weights[i] += this.learningRate * error * inputs[i];
}
}
}
this.predict = function (inputs) {
// transfer function
let sumProducts = 0;
for (let i = 0; i < inputs.length; i += 1) {
sumProducts += inputs[i] * weights[i];
}
// activation function, threshold = 1
return (sumProducts >= 1) ? 1 : 0; // <-- this is the line I have a question about
}
}
// train
let data = [
[0, 0, 0],
[0, 1, 0],
[1, 0, 0],
[1, 1, 1]
];
let p = new Perceptron(2);
const epochs = 20;
for (let i = 0; i < epochs; i += 1) {
let r = Math.floor(Math.random() * data.length);
p.train(data[r].slice(0, 2), data[r].slice(-1));
}
// test
for (let i = 0; i < data.length; i += 1) {
let inputs = data[i].slice(0, 2);
console.log(`inputs = ${inputs}; output = ${p.predict(inputs)}`);
}
}());

Answer 1

Your perceptron lacks a bias term, your equation is of form SUM_i w_i x_i, instead of SUM_i w_i x_i + b. With the functional form you have it is impossible to separate points, where the separating hyperplane does not cross the origin (and yours does not). Alternatively you can add a column of "1s" to your data, it will serve the same purpose, as the corresponding w_i will just behave as b (since all x_i will be 1)

Answer 2

For anyone else following this question, here is the final code. Thanks to #lejlot for helping!
/*jslint for:true, long:true, devel:true, this:true */
/*jshint esversion: 6*/
(function () {
"use strict";
function Perceptron(numInputs = 2) {
this.weights = Array.from({length: numInputs}, () => 2 * Math.random() - 1); // [-1, 1)
this.learningRate = 0.5;
this.bias = 1;
this.train = function (inputs, goal) {
// feed forward
let guess = this.predict(inputs);
// back propagation
let error = goal - guess;
if (error !== 0) {
for (let i = 0; i < this.weights.length; i += 1) {
this.weights[i] += error * inputs[i] * this.learningRate;
}
// Bias should be learned, not fixed. The update rules is
// identical to the weight, just lacks "* inputs[i]" (as
// described in the answer it is equivalent to having a constant
// input of 1, and multiplying by 1 does nothing).
this.bias += error * this.learningRate;
}
};
this.predict = function (inputs) {
// transfer function
let sumProducts = 0;
for (let i = 0; i < inputs.length; i += 1) {
sumProducts += inputs[i] * this.weights[i];
}
// Your perception lacks a bias term, your equation if of form
// SUM_i w_i x_i, instead of SUM_i w_i x_i + b. With the
// functional form you have it is impossible to separate points,
// where the separating hyperplane does not cross the origin
// (and yours does not). Alternatively you can add a column of
// "1s" to your data, it will serve the same purpose, as the
// corresponding w_i will just behave as "b" (since all x_i will
// be 1)
sumProducts += this.bias;
// activation function
return (sumProducts >= 0) ? 1 : 0;
};
}
// train - boolean AND
let data = [
[0, 0, 0],
[0, 1, 0],
[1, 0, 0],
[1, 1, 1]
];
let p = new Perceptron(2);
const epochs = 100;
for (let i = 0; i < epochs; i += 1) {
let r = Math.floor(Math.random() * data.length);
p.train(data[r].slice(0, 2), data[r].slice(-1));
}
// test
for (let i = 0; i < data.length; i += 1) {
let inputs = data[i].slice(0, 2);
console.log(`inputs = ${inputs}; output = ${p.predict(inputs)}`);
}
}());