TensorFlow restoring from NN does not work - neural-network

I am struggling with restoring values from NN in tensorflow. I tried to follow the examples on net, and here is my code:
import tensorflow as tf
import numpy as np
import math, random
import matplotlib.pyplot as plt
np.random.seed(1000) # for repro
function_to_learn = lambda x: np.sin(x) + 0.1*np.random.randn(*x.shape)
NUM_HIDDEN_NODES = 2
NUM_EXAMPLES = 1000
TRAIN_SPLIT = .8
MINI_BATCH_SIZE = 100
NUM_EPOCHS = 500
all_x = np.float32(np.random.uniform(-2*math.pi, 2*math.pi, (1, NUM_EXAMPLES))).T
np.random.shuffle(all_x)
train_size = int(NUM_EXAMPLES*TRAIN_SPLIT)
trainx = all_x[:train_size]
validx = all_x[train_size:]
trainy = function_to_learn(trainx)
validy = function_to_learn(validx)
plt.figure()
plt.scatter(trainx, trainy, c='green', label='train')
plt.scatter(validx, validy, c='red', label='validation')
plt.legend()
X = tf.placeholder(tf.float32, [None, 1], name="X")
Y = tf.placeholder(tf.float32, [None, 1], name="Y")
w_h = tf.Variable(tf.zeros([1, NUM_HIDDEN_NODES],name="w_h"))
b_h = tf.Variable(tf.zeros([1, NUM_HIDDEN_NODES],name="b_h"))
w_o = tf.Variable(tf.zeros([NUM_HIDDEN_NODES,1],name="w_o"))
b_o = tf.Variable(tf.zeros([1, 1],name="b_o"))
def init_weights(shape, init_method='xavier', xavier_params = (None, None)):
if init_method == 'zeros':
return tf.Variable(tf.zeros(shape, dtype=tf.float32))
elif init_method == 'uniform':
return tf.Variable(tf.random_normal(shape, stddev=0.01, dtype=tf.float32))
def model(X, num_hidden = NUM_HIDDEN_NODES):
w_h = init_weights([1, num_hidden], 'uniform' )
b_h = init_weights([1, num_hidden], 'zeros')
h = tf.nn.sigmoid(tf.matmul(X, w_h) + b_h)
w_o = init_weights([num_hidden, 1], 'xavier', xavier_params=(num_hidden, 1))
b_o = init_weights([1, 1], 'zeros')
return tf.matmul(h, w_o) + b_o
yhat = model(X, NUM_HIDDEN_NODES)
train_op = tf.train.AdamOptimizer().minimize(tf.nn.l2_loss(yhat - Y))
plt.figure()
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
for v in tf.all_variables():
print v.name
saver = tf.train.Saver()
errors = []
with tf.Session() as sess:
sess.run(tf.initialize_all_variables())
for i in range(NUM_EPOCHS):
for start, end in zip(range(0, len(trainx), MINI_BATCH_SIZE), range(MINI_BATCH_SIZE, len(trainx), MINI_BATCH_SIZE)):
sess.run(train_op, feed_dict={X: trainx[start:end], Y: trainy[start:end]})
mse = sess.run(tf.nn.l2_loss(yhat - validy), feed_dict={X:validx})
errors.append(mse)
if i%100 == 0:
print "epoch %d, validation MSE %g" % (i, mse)
print sess.run(w_h)
saver.save(sess,"/Python/tensorflow/res/save_net.ckpt", global_step = i)
print " ******* AFTR *******"
for v in tf.all_variables():
print v.name
plt.plot(errors)
plt.xlabel('#epochs')
plt.ylabel('MSE')
******* to get the restore values, I tried:**
import tensorflow as tf
import numpy as np
import math, random
import matplotlib.pyplot as plt
NUM_HIDDEN_NODES = 2
#SECOND PART TO GET THE STORED VALUES
w_h = tf.Variable(np.arange(NUM_HIDDEN_NODES).reshape(1, NUM_HIDDEN_NODES), dtype=tf.float32, name='w_h')
b_h = tf.Variable(np.arange(NUM_HIDDEN_NODES).reshape(1, NUM_HIDDEN_NODES), dtype=tf.float32, name='b_h')
w_o = tf.Variable(np.arange(NUM_HIDDEN_NODES).reshape(NUM_HIDDEN_NODES, 1), dtype=tf.float32, name='w_o')
b_o = tf.Variable(np.arange(1).reshape(1, 1), dtype=tf.float32, name='b_o')
saver = tf.train.Saver()
with tf.Session() as sess:
ckpt = tf.train.get_checkpoint_state("/Python/tensorflow/res/")
if ckpt and ckpt.model_checkpoint_path:
# Restores from checkpoint
saver.restore(sess, "/Python/tensorflow/res/save_net.ckpt-400")
print "Model loaded"
else:
print "No checkpoint file found"
print("weights:", sess.run(w_h))
print("biases:", sess.run(b_h))
Your help is greatly appreciated and I am almost giving up on this.
Thanks a lot again

It seems the checkpoint file you want to restore your variables from is different from the current variable/shape of existing code.
Save: (if substitute it with constants from definitions above)
w_h = tf.Variable(tf.zeros([1, 5],name="w_h"))
b_h = tf.Variable(tf.zeros([1, 5],name="b_h"))
w_o = tf.Variable(tf.zeros([5,1],name="w_o"))
b_o = tf.Variable(tf.zeros([1, 1],name="b_o"))
Restore:
w_h = tf.Variable(np.arange(10).reshape(1, 10), dtype=tf.float32, name='w_h')
b_h = tf.Variable(np.arange(10).reshape(1, 10), dtype=tf.float32, name='b_h')
w_o = tf.Variable(np.arange(10).reshape(10, 1), dtype=tf.float32, name='w_o')
b_o = tf.Variable(np.arange(1).reshape(1, 1), dtype=tf.float32, name='b_o')
To prevent these types of problems, try to use functions for training and inference so all your code will same variables and constants.

You are creating two sets of weights, once globally and second time when you call init_weights. The second set of variables is the one that's getting optimized, but both sets are saved.
In your eval code, you are creating this set of variables once, so your restore only restores the first set, which has not been modified after initialization.
The solution is to either factor out model creation code so that exactly same graph is created during training and during eval, or to use meta_graph which will recreate graph structure during restore.

Related

Pytorch-GPU what am I forgetting to move over to the GPU?

I'm getting this error. What am I leaving out, I feel that I have tired everything.
Also is there not a easy way to just use the GPU and not the CPU I feel like I have tried all those options as well. As in not using .cuda() everywhere
This is one of my first neutral networks so please go easy on me. (most of it is from Class)
RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cpu! (when checking arugment for argument target in method wrapper_nll_loss_forward)
import torch.cuda
import numpy as np
import time
from torchvision import datasets
import torchvision.transforms as transforms
import matplotlib.pyplot as plt
import torch.nn as nn
import torch.nn.functional as F
from torchsummary import summary
torch.cuda.set_device(0)
def load_data():
num_workers = 0
load_data.batch_size = 20
transform = transforms.ToTensor()
train_data = datasets.MNIST(root='data', train=True, download=True, transform=transform)
test_data = datasets.MNIST(root='data', train=False, download=True, transform=transform)
load_data.train_loader = torch.utils.data.DataLoader(train_data,
batch_size=load_data.batch_size, num_workers=num_workers, pin_memory=True)
test_loader = torch.utils.data.DataLoader(test_data,
batch_size=load_data.batch_size, num_workers=num_workers, pin_memory=True)
def visualize():
dataiter = iter(load_data.train_loader)
visualize.images, labels = dataiter.next()
visualize.images = visualize.images.numpy()
fig = plt.figure(figsize=(25, 4))
for idx in np.arange(load_data.batch_size):
ax = fig.add_subplot(2, load_data.batch_size/2, idx+1, xticks=[], yticks=[])
ax.imshow(np.squeeze(visualize.images[idx]), cmap='gray')
ax.set_title(str(labels[idx].item()))
#plt.show()
def fig_values():
img = np.squeeze(visualize.images[1])
fig = plt.figure(figsize = (12,12))
ax = fig.add_subplot(111)
ax.imshow(img, cmap='gray')
width, height = img.shape
thresh = img.max()/2.5
for x in range(width):
for y in range(height):
val = round(img[x][y],2) if img[x][y] !=0 else 0
ax.annotate(str(val), xy=(y,x),
horizontalalignment='center',
verticalalignment='center',
color='white' if img[x][y]<thresh else 'black')
#plt.show()
load_data()
visualize()
fig_values()
class NeuralNet(nn.Module):
def __init__(self, gpu = True):
super(NeuralNet, self ).__init__()
self.fc1 = nn.Linear(28 * 28, 16).cuda()
self.fc2 = nn.Linear(16, 10).cuda()
def forward(self, x):
x = x.view(-1, 28 * 28).cuda()
x = F.relu(self.fc1(x)).cuda()
x = self.fc2(x).cuda()
return x
def training():
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
n_epochs = 100
model.train().cuda()
for epoch in range(n_epochs):
train_loss = 0.0
for data, target in load_data.train_loader:
optimizer.zero_grad()
output = model(data)
loss = criterion(output, target)
loss.backward()
optimizer.step()
train_loss += loss.item()*data.size(0)
train_loss = train_loss/len(load_data.train_loader.dataset)
print('Epoch: {} \tTraining Loss: {:.6f}'.format(
epoch+1,
train_loss
))
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = NeuralNet().to(device)
summary(model, input_size=(1, 28, 28))
training()
Your data, and target are not in GPU (considering the removal of repeated cuda calls).
You are also doing of unnecessary cuda() which is not needed. Simply, see where your data and model are. Take the model to GPU, take the data and label to GPU, finally feed data to the model.
Don't use, cuda(), use to.device(), it's safer in the long run and easily customizable in multi-gpu setup.
import torch.cuda
import numpy as np
import time
from torchvision import datasets
import torchvision.transforms as transforms
import matplotlib.pyplot as plt
import torch.nn as nn
import torch.nn.functional as F
from torchsummary import summary
torch.cuda.set_device(0)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
def load_data():
num_workers = 0
load_data.batch_size = 20
transform = transforms.ToTensor()
train_data = datasets.MNIST(root='data', train=True, download=True, transform=transform)
test_data = datasets.MNIST(root='data', train=False, download=True, transform=transform)
load_data.train_loader = torch.utils.data.DataLoader(train_data,
batch_size=load_data.batch_size, num_workers=num_workers, pin_memory=True)
test_loader = torch.utils.data.DataLoader(test_data,
batch_size=load_data.batch_size, num_workers=num_workers, pin_memory=True)
def visualize():
dataiter = iter(load_data.train_loader)
visualize.images, labels = dataiter.next()
visualize.images = visualize.images.numpy()
fig = plt.figure(figsize=(25, 4))
for idx in np.arange(load_data.batch_size):
ax = fig.add_subplot(2, load_data.batch_size/2, idx+1, xticks=[], yticks=[])
ax.imshow(np.squeeze(visualize.images[idx]), cmap='gray')
ax.set_title(str(labels[idx].item()))
#plt.show()
def fig_values():
img = np.squeeze(visualize.images[1])
fig = plt.figure(figsize = (12,12))
ax = fig.add_subplot(111)
ax.imshow(img, cmap='gray')
width, height = img.shape
thresh = img.max()/2.5
for x in range(width):
for y in range(height):
val = round(img[x][y],2) if img[x][y] !=0 else 0
ax.annotate(str(val), xy=(y,x),
horizontalalignment='center',
verticalalignment='center',
color='white' if img[x][y]<thresh else 'black')
#plt.show()
load_data()
visualize()
fig_values()
class NeuralNet(nn.Module):
def __init__(self, gpu = True):
super(NeuralNet, self ).__init__()
self.fc1 = nn.Linear(28 * 28, 16)
self.fc2 = nn.Linear(16, 10)
def forward(self, x):
x = x.view(-1, 28 * 28)
x = F.relu(self.fc1(x))
x = self.fc2(x)
return x
def training():
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
n_epochs = 100
model.train()
for epoch in range(n_epochs):
train_loss = 0.0
for data, target in load_data.train_loader:
optimizer.zero_grad()
###################################
data = data.to(device)
target = target.to(device)
###################################
output = model(data)
loss = criterion(output, target)
loss.backward()
optimizer.step()
train_loss += loss.item()*data.size(0)
train_loss = train_loss/len(load_data.train_loader.dataset)
print('Epoch: {} \tTraining Loss: {:.6f}'.format(
epoch+1,
train_loss
))
model = NeuralNet().to(device)
summary(model, input_size=(1, 28, 28))
training()
Clearly your target variable is not on GPU.
Also, its a bad idea to call .cuda() inside forward()
def forward(self, x):
x = x.view(-1, 28 * 28).cuda() # BAD
x = F.relu(self.fc1(x)).cuda() # BAD
x = self.fc2(x).cuda() #BAD
return x
Rather, remove all .cuda() inside forward and do this is main loop
for data, target in load_data.train_loader:
data = data.cuda()
target = target.cuda()

Coefficient of determination is close to -1

I am down loading stock data from you finance.
I am trying to generate stock trading signal using ANN.
I am getting Coefficient of determination is close to -1 and prediction is seem to be mirror image.
Can someone suggest.
I am getting similar Coefficient of determination with others ML.
It seems to be strange.
import numpy as np
import yfinance as yf
import talib as ta
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, r2_score, accuracy_score
from sklearn.metrics import classification_report
from sklearn.preprocessing import StandardScaler
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
# Ignore warnings
import warnings
warnings.filterwarnings('ignore')
import random
random.seed(42)
# YYYY-MM-DD
start_date = '2010-01-01'
end_date = '2020-08-03'
#df = yf.download(tickers="^NSEI", start=start_date, end=end_date, interval="1d", progress=False)
df = yf.download("SBIN.NS", start=start_date, end=end_date, interval="1d", progress=False)
def create_trading_condition(df):
df['RSI'] = ta.RSI(df['Adj Close'].values, timeperiod = 9)
df['MACD'] = ta.MACD(df['Adj Close'].values, fastperiod=12, slowperiod=26, signalperiod=9)[0]
df['Williams %R'] = ta.WILLR(df['High'].values, df['Low'].values, df['Adj Close'].values, 7)
df['C-O'] = df['Adj Close'] - df['Open']
df['H-L'] = df['High'] - df['Low']
df['STDEV']= df['Adj Close'].rolling(5).std()
df['TARGET'] = np.where(df['Adj Close'].shift(-1) > df['Adj Close'], 1, 0)
df = df.dropna()
X = df[['RSI', 'MACD', 'Williams %R', 'C-O', 'H-L', 'STDEV']]
Y = df['TARGET']
#print(df)
return (df, X, Y)
df_new, X, Y = create_trading_condition(df)
X_train,X_test,Y_train,Y_test = train_test_split(X, Y, shuffle=False, train_size=0.8)
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)
clf = Sequential()
clf.add(Dense(units = 128, kernel_initializer = 'uniform', activation= 'relu', input_dim = X.shape[1]))
clf.add(Dense(units = 128, kernel_initializer = 'uniform', activation= 'relu'))
clf.add(Dense(units = 1, kernel_initializer = 'uniform', activation= 'sigmoid'))
clf.compile(optimizer = 'adam', loss = 'mean_squared_error', metrics= ['accuracy'])
clf.fit(X_train, Y_train, batch_size = 10, epochs = 100)
Y_pred = clf.predict(X_test)
#print(Y_pred)
Y_pred = (Y_pred > 0.5)
df_new['Y_pred'] = np.NaN
df_new.iloc[(len(df_new) - len(Y_pred)):,-1:] = Y_pred
trade_df = df_new.dropna()
trade_df['Tomorrows Returns'] = 0.
trade_df['Tomorrows Returns'] = np.log(trade_df['Adj Close']/trade_df['Adj Close'].shift(1))
trade_df['Tomorrows Returns'] = trade_df['Tomorrows Returns'].shift(-1)
# Y_pred = true for long position otherwise short position
trade_df['Strategy Returns'] = 0.
trade_df['Strategy Returns'] = np.where(trade_df['Y_pred'] == True, trade_df['Tomorrows Returns'], -trade_df['Tomorrows Returns'])
trade_df['Cumulative Market Returns'] = np.cumsum(trade_df['Tomorrows Returns'])
trade_df['Cumulative Strategy Returns'] = np.cumsum(trade_df['Strategy Returns'])
#accuracy_test = accuracy_score(Y_test, clf.predict(X_test))
# mean squared error
mse_test = mean_squared_error(Y_test, clf.predict(X_test))
# Coefficient of determination
#r2_test = r2_score(Y_test, clf.predict(X_test))
r2_test = r2_score(Y_test, Y_pred)
#report_test = classification_report(Y_test, clf.predict(X_test))
#print("Test accuracy score: %.2f" % accuracy_test)
print("Test mean squared error: %.2f" % mse_test)
print('Test R-square: %.2f\n' % r2_test)
#print(report_test)
import matplotlib.pyplot as plt
plt.figure(figsize=(10,5))
plt.plot(trade_df['Cumulative Market Returns'], color='r', label='Market Returns')
plt.plot(trade_df['Cumulative Strategy Returns'], color='g', label='Strategy Returns')
plt.legend()
plt.show()

L1 regulariser Pytorch acting opposite to what I expect

I'm trying to add an L1 penalty to a specific layer of a neural network, and I have the code below (in which I attempt to add l1 penalty to the first layer). If I run it for lambda = 0 (i.e. no penalty), the output gets very close to the expected weights those being [10, 12, 2, 11, -0.25]) and if I run for enough epochs or reduce batch size it will get it exactly, as in the output below:
mlp.0.weight
Parameter containing:
tensor([[ 9.8657, -11.8305, 2.0242, 10.8913, -0.1978]],
requires_grad=True)
Then, when I run it for a large lambda, say 1000, I would expect these weights to shrink towards zero as there is a large penalty being added to the loss that we are trying to minimise. However, the opposite happens and the weights explode, as in the output below (for lam = 1000)
mlp.0.weight
Parameter containing:
tensor([[-13.9368, 9.9072, 2.2447, -11.6870, 26.7293]],
requires_grad=True)
If anyone could help me, that'd be great. I'm new to pytorch (but not the idea of regularisation), so I'm guessing it's something in my code that is the problem.
Thanks
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
import numpy as np
from sklearn.linear_model import LinearRegression
class TrainDataset(Dataset):
def __init__(self, data):
self.data = data
def __len__(self):
return self.data.shape[0]
def __getitem__(self, ind):
x = self.data[ind][1:]
y = self.data[ind][0]
return x, y
class TestDataset(TrainDataset):
def __getitem__(self, ind):
x = self.data[ind]
return x
torch.manual_seed(94)
x_train = np.random.rand(1000, 5)
y_train = x_train[:, 0] * 10 - x_train[:, 1] * 12 + x_train[:, 2] * 2 + x_train[:, 3] * 11 - x_train[:, 4] * 0.25
y_train = y_train.reshape(1000, 1)
x_train.shape
y_train.shape
train_data = np.concatenate((y_train, x_train), axis=1)
train_set = TrainDataset(train_data)
batch_size = 100
train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True)
class MLP(nn.Module):
def __init__(self):
super(MLP, self).__init__()
self.mlp = nn.Sequential(nn.Linear(5, 1, bias=False))
def forward(self, x_mlp):
out = self.mlp(x_mlp)
return out
device = 'cpu'
model = MLP()
optimizer = torch.optim.SGD(model.parameters(), lr=0.02, momentum=0.82)
criterion = nn.MSELoss()
epochs = 5
lam = 0
model.train()
for epoch in range(epochs):
losses = []
for batch_num, input_data in enumerate(train_loader):
optimizer.zero_grad()
x, y = input_data
x = x.to(device).float()
y = y.reshape(batch_size, 1)
y = y.to(device)
output = model(x)
for name, param in model.named_parameters():
if name == 'mlp.0.weight':
l1_norm = torch.norm(param, 1)
loss = criterion(output, y) + lam * l1_norm
loss.backward()
optimizer.step()
print('\tEpoch %d | Batch %d | Loss %6.2f' % (epoch, batch_num, loss.item()))
for name, param in model.named_parameters():
if param.requires_grad:
print(name)
print(param)
I found that if I use Adagrad as the optimiser instead of SGD, it acts as expected. Will need to look into the difference of those now, but this can be considered answered.

the model learned, but cannot predict?

I follow the tutorial to write some linear regression code about boston price, it worked very well and the loss became smaller, when I wanted to paint the graph in matplotlib, I found the graph not showed as what in my mind.
I searched,but could not solve my question.
import pandas as pd
from sklearn.datasets import load_boston
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
import torch
from torch.autograd import Variable
import matplotlib.pyplot as plt
if __name__ == '__main__':
boston = load_boston()
col_names = ['feature_{}'.format(i) for i in range(boston['data'].shape[1])]
df_full = pd.DataFrame(boston['data'], columns=col_names)
scalers_dict = {}
for col in col_names:
scaler = StandardScaler()
df_full[col] = scaler.fit_transform(df_full[col].values.reshape(-1, 1))
scalers_dict[col] = scaler
x_train, x_test, y_train, y_test = train_test_split(df_full.values, boston['target'], test_size=0.2, random_state=2)
model = torch.nn.Sequential(torch.nn.Linear(x_train.shape[1], 1), torch.nn.ReLU())
criterion = torch.nn.MSELoss(reduction='mean')
optimizer = torch.optim.Adam(model.parameters(), lr=0.1)
n_epochs = 2000
train_loss = []
test_loss = []
x_train = Variable(torch.from_numpy(x_train).float(), requires_grad=True)
y_train = Variable(torch.from_numpy(y_train).float())
for epoch in range(n_epochs):
y_hat = model(x_train)
loss = criterion(y_hat, y_train)
optimizer.zero_grad()
loss.backward()
optimizer.step()
epoch_loss = loss.data ** (1/2)
train_loss.append(epoch_loss)
if (epoch + 1) % 250 == 0:
print("{}:loss = {}".format(epoch + 1, epoch_loss))
order = y_train.argsort()
y_train = y_train[order]
x_train = x_train[order, :]
model.eval()
predicted = model(x_train).detach().numpy()
actual = y_train.numpy()
print('predicted:", predicted[:5].flatten(), actual[:5])
plt.plot(predicted.flatten(), 'r-', label='predicted')
plt.plot(actual, 'g-', label='actual')
plt.show()
why the predict were the same result like [22.4413, 22.4413, ...],
in the picture, it's a horizontal line.
I'm a very beginner to deeplearning, thank you very much for your help!

How to accumulate and appy gradients for Async n-step DQNetwork update in Tensorflow?

I am trying to implement Asynchronous Methods for Deep Reinforcement Learning and one of the steps requires to accumulate the gradient over different steps and then apply it.
What is the best way to achieve this in tensorflow?
I got so far as to accumulate the gradient and I don't think is the fastest way to achieve it (lots of transfers from tensorflow to python and back).
Any suggestions are welcome.
This is my code of a toy NN. It does not model or compute anything it just exercise the operations that I want to use.
import tensorflow as tf
from model import *
graph = tf.Graph()
with graph.as_default():
state = tf.placeholder(tf.float32, shape=[None, 80,80,1])
with tf.variable_scope('layer1'):
W = weight_variable([8, 8, 1, 32])
variable_summaries(W, "layer1/W")
b = bias_variable([32])
variable_summaries(b, "layer1/b")
h = conv2d(state, W, 4) + b
activation = tf.nn.relu(h)
pool1 = max_pool_2x2(activation)
print(pool1.get_shape())
pool1 = tf.reshape(pool1, [-1, 3200])
with tf.variable_scope('readout'):
W = weight_variable([3200, 3])
b = bias_variable([3])
logits = tf.matmul(pool1, W) + b
variable_summaries(h, "y")
action_indexes = tf.placeholder(tf.int32, shape=[None], name="action_indexes")
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, action_indexes)
starter_learning_rate = 1e-6
global_step = tf.Variable(0, trainable=False)
# decay every 1000 steps with a base of 0.96:
learning_rate = tf.train.exponential_decay(starter_learning_rate,
global_step,
10000, 0.96, staircase=True)
optimizer = tf.train.RMSPropOptimizer(learning_rate)
gradients_and_variables = optimizer.compute_gradients(loss, tf.trainable_variables())
discounted_values = tf.placeholder(tf.float32, shape=[None, 1])
with tf.Session(graph=graph) as s:
for v in tf.trainable_variables():
print(v.name, v.dtype, v.get_shape())
s.run(tf.initialize_all_variables())
feed_dict= {
state : np.zeros([1, 80, 80, 1]),
action_indexes: [1],
}
var_to_grad = dict((var.name, grad) for grad, var in gradients_and_variables)
keys = sorted(var_to_grad.keys())
print(keys)
name_to_var = dict((var.name, var) for _, var in gradients_and_variables)
for i in range(10):
gradients = s.run([ var_to_grad[k] for k in keys], feed_dict=feed_dict)
for k,v in zip(keys, gradients):
var_to_grad[k] += v
for k in keys:
print(var_to_grad[k])
s.run( optimizer.apply_gradients( (g, name_to_var[v]) for v,g in var_to_grad.iteritems()), feed_dict=feed_dict)
Updated code after #yaroslave suggestion:
import tensorflow as tf
from model import *
graph = tf.Graph()
with graph.as_default():
minibatch = 32
state = tf.placeholder(tf.float32, shape=[minibatch, 80,80,1], name="input")
with tf.variable_scope('layer1'):
W = weight_variable([8, 8, 1, 32])
variable_summaries(W, "layer1/W")
b = bias_variable([32])
variable_summaries(b, "layer1/b")
h = conv2d(state, W, 4) + b
activation = tf.nn.relu(h)
pool1 = max_pool_2x2(activation)
print(pool1.get_shape())
pool1 = tf.reshape(pool1, [-1, 3200])
with tf.variable_scope('readout'):
W = weight_variable([3200, 3])
b = bias_variable([3])
logits = tf.matmul(pool1, W) + b
variable_summaries(h, "y")
action_indexes = tf.placeholder(tf.int32, shape=[minibatch], name="action_indexes")
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, action_indexes)
starter_learning_rate = 1e-6
global_step = tf.Variable(0, trainable=False)
# decay every 1000 steps with a base of 0.96:
learning_rate = tf.train.exponential_decay(starter_learning_rate,
global_step,
10000, 0.96, staircase=True)
optimizer = tf.train.RMSPropOptimizer(learning_rate)
trainable_variables = tf.trainable_variables()
varname_to_var = dict( (v.name, v) for v in trainable_variables )
keys = sorted(varname_to_var.keys())
gradients_and_variables = optimizer.compute_gradients(loss, [ varname_to_var[k] for k in keys])
var_to_grad = dict((var.name, grad) for grad, var in gradients_and_variables)
name_to_var = dict((var.name, var) for _, var in gradients_and_variables)
# save the gradients in memory
var_to_ref_grad = {}
for k in keys:
grad = var_to_grad[k]
print(k, grad.get_shape())
ref = tf.Variable(tf.zeros_like(grad))
ref = ref.assign_add(grad)
var_to_ref_grad[k] = ref
discounted_values = tf.placeholder(tf.float32, shape=[None, 1], name='discounted_values')
# control when to apply gradients
compute_gradients_flag = tf.placeholder(tf.int32, name="compute_gradients")
def fn1():
var_grad_list = []
for k in keys:
grad = var_to_ref_grad[k]
var = varname_to_var[k]
var_grad_list.append((grad,var))
optimizer.apply_gradients(var_grad_list)
return tf.no_op()
fn2 = lambda : tf.no_op()
last_op = tf.cond(tf.equal(compute_gradients_flag, 1), fn1, fn2)
with tf.Session(graph=graph) as s:
feed_dict= {
state : np.zeros([minibatch, 80, 80, 1]),
action_indexes: [1],
compute_gradients_flag: False,
}
s.run(tf.initialize_all_variables())
for i in range(10):
# accumulate gradients
s.run(last_op, feed_dict=feed_dict)
You don't really have to manually accumulate gradients. You can have Tensorflow accumulate them for you by applying the rollout update as a batch.
s_list = list_of_states_visited
a_list = list_of_actions_taken
R_list = list_of_value_targets
sess.run(local_net.update, feed_dict={
local_net.input: s_list,
local_net.a: a_list,
local_net.R: R_list
})
Something like this might work to create ops for accumulating gradients, resetting the accumulated gradients, and applying the accumulated gradients (untested!):
def build_gradient_accumulators(optimizer, gradients_and_variables):
accum_grads_and_vars = []
accumulators = []
resetters = []
for grad, var in gradients_and_variables:
accum = tf.Variable(tf.zeros_like(grad))
accum = accum.assign_add(grad)
accumulators.append(accum)
accum_grads_and_vars.append((accum, var))
resetters.append(tf.assign(accum, tf.zeros_like(accum)))
reset_op = tf.group(*resetters)
accum_op = tf.group(*accumulators)
apply_op = optimizer.apply_gradients(accum_grads_and_vars)
return reset_op, accum_op, apply_op