Subprocessing Data Loading in pytroch into Google Colab - subprocess

Question

I'm working on training a deep neural network using pytorch and I use DataLoader for preprocessing data and multi-processing purpose over dataset. I set num_workers attribute to positive number like 4 and my batch_size is 8. I train network on Google Colab Environment but when training keep on after few minutes, stop training and get error in reading .PNG files. I think it's memory error and I want to know what is relation between number of GPU and batch_size and num_workers to set up a reasonable relation between them specially in Google Colab .

Answer 1

I think you can follow this page:
https://colab.research.google.com/notebook#fileId=1jxUPzMsAkBboHMQtGyfv5M5c7hU8Ss2c&scrollTo=EM7EnBoyK8nR
It provide a guide of how to set settings of Google Colab.
I try it and feels really fast.
Hope you love it.
Following is the code it provides but I change a bit about install pytorch:
#!/usr/bin/env python
# encoding: utf-8
import sys
sys.version
# http://pytorch.org/
from os import path
from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag
platform = '{}{}-{}'.format(get_abbr_impl(), get_impl_ver(), get_abi_tag())
accelerator = 'cu80' if path.exists('/opt/bin/nvidia-smi') else 'cpu'
!pip install -q http://download.pytorch.org/whl/{accelerator}/torch-0.3.0.post4-{platform}-linux_x86_64.whl torchvision
import torch
import torch.nn as nn
import torchvision.datasets as dsets
import torchvision.transforms as transforms
from torch.autograd import Variable
input_size = 784 # The image size = 28 x 28 = 784
hidden_size = 500 # The number of nodes at the hidden layer
num_classes = 10 # The number of output classes. In this case, from 0 to 9
num_epochs = 5 # The number of times entire dataset is trained
batch_size = 100 # The size of input data took for one iteration
learning_rate = 1e-3 # The speed of convergence
train_dataset = dsets.MNIST(root='./data',
train=True,
transform=transforms.ToTensor(),
download=True)
test_dataset = dsets.MNIST(root='./data',
train=False,
transform=transforms.ToTensor())
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True)
test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
batch_size=batch_size,
shuffle=False)
class Net(nn.Module):
def __init__(self, input_size, hidden_size, num_classes):
super(Net, self).__init__() # Inherited from the parent class nn.Module
self.fc1 = nn.Linear(input_size, hidden_size) # 1st Full-Connected Layer: 784 (input data) -> 500 (hidden node)
self.relu = nn.ReLU() # Non-Linear ReLU Layer: max(0,x)
self.fc2 = nn.Linear(hidden_size, num_classes) # 2nd Full-Connected Layer: 500 (hidden node) -> 10 (output class)
def forward(self, x): # Forward pass: stacking each layer together
out = self.fc1(x)
out = self.relu(out)
out = self.fc2(out)
return out
net = Net(input_size, hidden_size, num_classes)
use_cuda = True
if use_cuda and torch.cuda.is_available():
net.cuda()
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate)
for epoch in range(num_epochs):
for i, (images, labels) in enumerate(train_loader): # Load a batch of images with its (index, data, class)
images = Variable(images.view(-1, 28*28)) # Convert torch tensor to Variable: change image from a vector of size 784 to a matrix of 28 x 28
labels = Variable(labels)
if use_cuda and torch.cuda.is_available():
images = images.cuda()
labels = labels.cuda()
optimizer.zero_grad() # Intialize the hidden weight to all zeros
outputs = net(images) # Forward pass: compute the output class given a image
loss = criterion(outputs, labels) # Compute the loss: difference between the output class and the pre-given label
loss.backward() # Backward pass: compute the weight
optimizer.step() # Optimizer: update the weights of hidden nodes
if (i+1) % 100 == 0: # Logging
print('Epoch [%d/%d], Step [%d/%d], Loss: %.4f'
%(epoch+1, num_epochs, i+1, len(train_dataset)//batch_size, loss.data[0]))
correct = 0
total = 0
for images, labels in test_loader:
images = Variable(images.view(-1, 28*28))
if use_cuda and torch.cuda.is_available():
images = images.cuda()
labels = labels.cuda()
outputs = net(images)
_, predicted = torch.max(outputs.data, 1) # Choose the best class from the output: The class with the best score
total += labels.size(0) # Increment the total count
correct += (predicted == labels).sum() # Increment the correct count
print('Accuracy of the network on the 10K test images: %d %%' % (100 * correct / total))
torch.save(net.state_dict(), 'fnn_model.pkl')