Neupy custom activation function - neural-network

Question

I am trying to create a custom activation layer base on the MNIST example in Neupy. However, once I apply my custom layer to the network, it stops training. For my custom function, I want to make the input value from a floating point value to a fixed point for both ReLU and Softmax function. Therefore, I create a function call "float_limit", which helps me to change a floating point value to be a fixed point value. My first idea is to use an int() function within my float_limit function. However, it shows type error since int() cannot use for tensor variable. So I change the int() function to be T.floor(), which can do the same work as int(). For the ReLU function works fine after applying the float_limit(). But once apply softmax function, the network stop training. May I ask that how can I fix this problem?
This is my code:
from sklearn import datasets, model_selection
from sklearn.preprocessing import OneHotEncoder
from neupy import environment,algorithms, layers
import numpy as np
from sklearn.model_selection import train_test_split
import theano
import theano.tensor as T
# load data
mnist = datasets.fetch_mldata('MNIST original')
data, target = mnist.data, mnist.target
# make one hot
data = data / 255.
data = data - data.mean(axis=0)
target_scaler = OneHotEncoder()
target = target_scaler.fit_transform(target.reshape((-1, 1)))
target = target.todense()
# split data for training and testing
environment.reproducible()
x_train, x_test, y_train, y_test = train_test_split(
data.astype(np.float32),
target.astype(np.float32),
train_size=(6. / 7)
)
# Theano is a main backend for the Gradient Descent based algorithms in NeuPy.
theano.config.floatX = 'float32'
#################### create new transfer function ###########################
################# float limit #####################
# # idea code
# def float_limit(n, b):
# d = 2 ** b
# return int(n * d) / d
def float_limit(n, b):
d = T.floor(2.0) ** b
return T.floor(n * d) / d
###################################################
################ custom function ##################
################## relu ##################
def relu(x, alpha=0):
if alpha == 0:
x = float_limit(x, 8)
result = 0.5 * (x + abs(x))
return result
else:
x = float_limit(x, 8)
alpha = T.tensor.as_tensor_variable(alpha)
f1 = 0.5 * (1 + alpha)
f2 = 0.5 * (1 - alpha)
return f1 * x + f2 * abs(x)
class custom_relu(layers.ActivationLayer):
def activation_function(self, input_value):
return relu(input_value)
#################### softmax ########################
class custom_softmax(layers.ActivationLayer):
def activation_function(self, input_value):
input_value = float_limit(input_value,8)
return T.nnet.softmax(input_value)
########### start the model architecture ############
network = algorithms.Momentum(
[
layers.Input(784),
custom_relu(500), #Relu
# Squared(300),
custom_relu(300),
# layers.Relu(300), #Relu
custom_softmax(10), #Softmax
# layers.Sigmoid
# layers.Input(784),
# tansig(500),
# tansig(500),
],
error='categorical_crossentropy',
step=0.01,
verbose=True,
shuffle_data=True,
momentum=0.99,
nesterov=True,
)
# print the architecture(Input shape, Layer Type, Output shape)
network.architecture()
# train the network
network.train(x_train, y_train, x_test, y_test, epochs=30)
# show the accuracy
from sklearn import metrics
y_predicted = network.predict(x_test).argmax(axis=1)
y_test = np.asarray(y_test.argmax(axis=1)).reshape(len(y_test))
print("y_predicted",y_predicted)
print("y_test",y_test)
print(metrics.classification_report(y_test, y_predicted))
score = metrics.accuracy_score(y_test, y_predicted)
print("Validation accuracy: {:.2%}".format(score))
# plot the image
from neupy import plots
plots.error_plot(network)