Related
I am now reproducing the achievements of an elder：1:https://github.com/sharathadavanne/seld-net.The source code is designed based on CPU. Now I want to run this code on GPU, but there is a problem of memory explosion. I'm a beginner of tensorflow. I don't know what to do now. Can any kind person give me some advice。
#
# A wrapper script that trains the SELDnet and SELD-TCN.
# The training stops when the SELD error (check paper) stops improving.
#
import gc
import tensorflow
import os
import sys
import numpy as np
import matplotlib.pyplot as plot
import cls_data_generator
import evaluation_metrics
import keras_model
import parameter
import utils
import time
plot.switch_backend('agg')
def set_gpu():
"""GPU相关设置"""
# 打印变量在那个设备上
# tf.debugging.set_log_device_placement(True)
# 获取物理GPU个数
gpus = tensorflow.config.experimental.list_physical_devices('GPU')
print('物理GPU个数为：', len(gpus))
# 设置内存自增长
for gpu in gpus:
tensorflow.config.experimental.set_memory_growth(gpu, True)
print('-------------已设置完GPU内存自增长--------------')
# 获取逻辑GPU个数
logical_gpus = tensorflow.config.experimental.list_logical_devices('GPU')
print('逻辑GPU个数为：', len(logical_gpus))
def collect_test_labels(_data_gen_test, _data_out, classification_mode, quick_test):
# Collecting ground truth for test data
nb_batch = 2 if quick_test else _data_gen_test.get_total_batches_in_data()
batch_size = _data_out[0][0]
gt_sed = np.zeros((nb_batch * batch_size, _data_out[0][1], _data_out[0][2]))
gt_doa = np.zeros((nb_batch * batch_size, _data_out[0][1], _data_out[1][2]))
print("nb_batch in test: {}".format(nb_batch))
cnt = 0
for tmp_feat, tmp_label in _data_gen_test.generate():
gt_sed[cnt * batch_size:(cnt + 1) * batch_size, :, :] = tmp_label[0]
gt_doa[cnt * batch_size:(cnt + 1) * batch_size, :, :] = tmp_label[1]
cnt = cnt + 1
if cnt == nb_batch:
break
return gt_sed.astype(int), gt_doa
def plot_functions(fig_name, _tr_loss, _val_loss, _sed_loss, _doa_loss, _epoch_metric_loss):
plot.figure()
nb_epoch = len(_tr_loss)
plot.subplot(311)
plot.plot(range(nb_epoch), _tr_loss, label='train loss')
plot.plot(range(nb_epoch), _val_loss, label='val loss')
plot.legend()
plot.grid(True)
plot.subplot(312)
plot.plot(range(nb_epoch), _epoch_metric_loss, label='metric')
plot.plot(range(nb_epoch), _sed_loss[:, 0], label='er')
plot.plot(range(nb_epoch), _sed_loss[:, 1], label='f1')
plot.legend()
plot.grid(True)
plot.subplot(313)
plot.plot(range(nb_epoch), _doa_loss[:, 1], label='gt_thres')
plot.plot(range(nb_epoch), _doa_loss[:, 2], label='pred_thres')
plot.legend()
plot.grid(True)
plot.savefig(fig_name)
plot.close()
def main(argv):
"""
Main wrapper for training sound event localization and detection network.
:param argv: expects two optional inputs.
first input: job_id - (optional) all the output files will be uniquely represented with this. (default) 1
second input: task_id - (optional) To chose the system configuration in parameters.py.
(default) uses default parameters
"""
set_gpu()
if len(argv) != 3:
print('\n\n')
print('-------------------------------------------------------------------------------------------------------')
print('The code expected two inputs')
print('\t>> python seld.py <job-id> <task-id>')
print('\t\t<job-id> is a unique identifier which is used for output filenames (models, training plots). '
'You can use any number or string for this.')
print('\t\t<task-id> is used to choose the user-defined parameter set from parameter.py')
print('Using default inputs for now')
print('-------------------------------------------------------------------------------------------------------')
print('\n\n')
# use parameter set defined by user
task_id = '1' if len(argv) < 3 else argv[-1]
params = parameter.get_params(task_id)
job_id = 1 if len(argv) < 2 else argv[1]
model_dir = 'models/'
utils.create_folder(model_dir)
unique_name = '{}_ov{}_split{}_{}{}_3d{}_{}'.format(
params['dataset'], params['overlap'], params['split'], params['mode'], params['weakness'],
int(params['cnn_3d']), job_id
)
unique_name = os.path.join(model_dir, unique_name)
print("unique_name: {}\n".format(unique_name))
data_gen_train = cls_data_generator.DataGenerator(
dataset=params['dataset'], ov=params['overlap'], split=params['split'], db=params['db'], nfft=params['nfft'],
batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'],
weakness=params['weakness'], datagen_mode='train', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'],
azi_only=params['azi_only']
)
data_gen_test = cls_data_generator.DataGenerator(
dataset=params['dataset'], ov=params['overlap'], split=params['split'], db=params['db'], nfft=params['nfft'],
batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'],
weakness=params['weakness'], datagen_mode='test', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'],
azi_only=params['azi_only'], shuffle=False
)
data_in, data_out = data_gen_train.get_data_sizes()
print(
'FEATURES:\n'
'\tdata_in: {}\n'
'\tdata_out: {}\n'.format(
data_in, data_out
)
)
gt = collect_test_labels(data_gen_test, data_out, params['mode'], params['quick_test'])
sed_gt = evaluation_metrics.reshape_3Dto2D(gt[0])
doa_gt = evaluation_metrics.reshape_3Dto2D(gt[1])
print(
'MODEL:\n'
'\tdropout_rate: {}\n'
'\tCNN: nb_cnn_filt: {}, pool_size{}\n'
'\trnn_size: {}, fnn_size: {}\n'.format(
params['dropout_rate'],
params['nb_cnn3d_filt'] if params['cnn_3d'] else params['nb_cnn2d_filt'], params['pool_size'],
params['rnn_size'], params['fnn_size']
)
)
# SELD-TCN MODEL
print("DATA IN:" + str(data_in))
model = keras_model.get_seldtcn_model(data_in=data_in, data_out=data_out, dropout_rate=params['dropout_rate'],
nb_cnn2d_filt=params['nb_cnn2d_filt'], pool_size=params['pool_size'],
fnn_size=params['fnn_size'], weights=params['loss_weights'])
#'''
best_metric = 99999
conf_mat = None
best_conf_mat = None
best_epoch = -1
patience_cnt = 0
epoch_metric_loss = np.zeros(params['nb_epochs'])
tr_loss = np.zeros(params['nb_epochs'])
val_loss = np.zeros(params['nb_epochs'])
doa_loss = np.zeros((params['nb_epochs'], 6))
sed_loss = np.zeros((params['nb_epochs'], 2))
nb_epoch = 2 if params['quick_test'] else params['nb_epochs']
tot_time = 0
for epoch_cnt in range(nb_epoch):
start = time.time()
hist = model.fit_generator(
generator=data_gen_train.generate(),
steps_per_epoch=2 if params['quick_test'] else data_gen_train.get_total_batches_in_data(),
validation_data=data_gen_test.generate(),
validation_steps=2 if params['quick_test'] else data_gen_test.get_total_batches_in_data(),
epochs=1,
verbose=0
)
tr_loss[epoch_cnt] = hist.history.get('loss')[-1]
val_loss[epoch_cnt] = hist.history.get('val_loss')[-1]
pred = model.predict_generator(
generator=data_gen_test.generate(),
steps=2 if params['quick_test'] else data_gen_test.get_total_batches_in_data(),
verbose=2
)
if params['mode'] == 'regr':
sed_pred = evaluation_metrics.reshape_3Dto2D(pred[0]) > 0.5
doa_pred = evaluation_metrics.reshape_3Dto2D(pred[1])
sed_loss[epoch_cnt, :] = evaluation_metrics.compute_sed_scores(sed_pred, sed_gt, data_gen_test.nb_frames_1s())
if params['azi_only']:
doa_loss[epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xy(doa_pred, doa_gt,
sed_pred, sed_gt)
else:
doa_loss[epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xyz(doa_pred, doa_gt,
sed_pred, sed_gt)
epoch_metric_loss[epoch_cnt] = np.mean([
sed_loss[epoch_cnt, 0],
1-sed_loss[epoch_cnt, 1],
2*np.arcsin(doa_loss[epoch_cnt, 1]/2.0)/np.pi,
1 - (doa_loss[epoch_cnt, 5] / float(doa_gt.shape[0]))]
)
plot_functions(unique_name, tr_loss, val_loss, sed_loss, doa_loss, epoch_metric_loss)
patience_cnt += 1
if epoch_metric_loss[epoch_cnt] < best_metric:
best_metric = epoch_metric_loss[epoch_cnt]
best_conf_mat = conf_mat
best_epoch = epoch_cnt
model.save('{}_model.h5'.format(unique_name))
patience_cnt = 0
print(
'epoch_cnt: %d, time: %.2fs, tr_loss: %.2f, val_loss: %.2f, '
'F1_overall: %.2f, ER_overall: %.2f, '
'doa_error_gt: %.2f, doa_error_pred: %.2f, good_pks_ratio:%.2f, '
'error_metric: %.2f, best_error_metric: %.2f, best_epoch : %d' %
(
epoch_cnt, time.time() - start, tr_loss[epoch_cnt], val_loss[epoch_cnt],
sed_loss[epoch_cnt, 1], sed_loss[epoch_cnt, 0],
doa_loss[epoch_cnt, 1], doa_loss[epoch_cnt, 2], doa_loss[epoch_cnt, 5] / float(sed_gt.shape[0]),
epoch_metric_loss[epoch_cnt], best_metric, best_epoch
)
)
if epoch_cnt in [2, 10, 20, 30, 40, 50, 60, 70, 80, 100, 120, 140, 150, 170, 190, 200, 250, 300, 450, 400]:
print_metrics(best_conf_mat, best_epoch, best_metric, doa_loss, sed_gt, sed_loss)
tot_time += (time.time() - start)
print("Time elapsed: %.2f hrs\n" %(tot_time/3600))
if patience_cnt > params['patience']:
break
print_metrics(best_conf_mat, best_epoch, best_metric, doa_loss, sed_gt, sed_loss)
def print_metrics(best_conf_mat, best_epoch, best_metric, doa_loss, sed_gt, sed_loss):
print('best_conf_mat : {}'.format(best_conf_mat))
print('best_conf_mat_diag : {}'.format(np.diag(best_conf_mat)))
print('saved model for the best_epoch: {} with best_metric: {}, '.format(best_epoch, best_metric))
print('DOA Metrics: doa_loss_gt: {}, doa_loss_pred: {}, good_pks_ratio: {}'.format(
doa_loss[best_epoch, 1], doa_loss[best_epoch, 2], doa_loss[best_epoch, 5] / float(sed_gt.shape[0])))
print('SED Metrics: F1_overall: {}, ER_overall: {}'.format(sed_loss[best_epoch, 1], sed_loss[best_epoch, 0]))
if __name__ == "__main__":
try:
sys.exit(main(sys.argv))
except (ValueError, IOError) as e:
sys.exit(e)
I am using Google OR-Tools in python to solve a capacitated VRP with pickup/delivery. In many cases the solver works well and returns reasonable solutions, but we have found that for some data sets the solver will always return one job per truck regardless of the time involved for the route.
I have the model set up as follows:
My initial vehicle count is equal to the number of jobs in the data-set, and we allow OR-Tools to automatically minimize the truck count.
Each job's pickup location has a demand of 1 and each job's dropoff location has a demand of -1, to enforce delivery immediately after pickup.
We set the maximum drive time per vehicle to 8 hours.
Then, each job has an associated quantity attached for pickup, and we separate this job into multiple deliveries based on a truck's capacity. For instance, if a job requires 60 tons delivered, we represent that as three jobs at 20 tons each (the maximum a vehicle is allowed to carry on an interstate in the U.S)
Now, we have a simple data set with a pickup location at: 698 Longtown Rd, Columbia, SC and a dropoff location at: 121 Chappell Creek Rd Hopkins, SC. This is a drive time of 32 minutes, or a total trip time of 64 minutes. This job has an associated quantity of 60 tons, which will require 3 truck loads.
The results we receive from or-tools shows one load per truck, and this result does not change regardless of how long we allow the solver to run. The optimal solution would allow one truck to complete all loads, as this is still drastically under the 8 hour drive time limit.
Here is my code:
import json
import math
import traceback
import urllib
import redis
import requests
import boto3
from signal import signal, SIGINT, SIGTERM
from ortools.constraint_solver import pywrapcp, routing_enums_pb2
url = 'https://test-api.truckit.com/api/2/signin'
api_data = {"password": "", "username": ""}
response = requests.post(url, json=api_data)
api_data = response.json()
def build_auth_header(token):
header = {'Authorization': f'Token {token}'}
return header
class SignalHandler:
def __init__(self):
self.received_signal = False
signal(SIGINT, self._signal_handler)
signal(SIGTERM, self._signal_handler)
def _signal_handler(self, signal, frame):
print(f"handling signal {signal}, exiting gracefully")
self.received_signal = True
sqs = boto3.resource("sqs")
queue = sqs.get_queue_by_name(QueueName="")
redisClient = redis.Redis(host='', port=6379,
password='')
def create_distance_matrix(data):
addresses = data["addresses"]
API_key = data["API_key"]
origin_addresses = []
dest_addresses = addresses
distance_matrix = []
responses = {}
responses['destination_addresses'] = []
responses['origin_addresses'] = []
responses['rows'] = []
# Send q requests, returning max_rows rows per request.
for i in range(0, len(addresses)):
origin_addresses.clear()
origin_addresses.append(addresses[i])
for j in range(0, len(addresses), 25):
dest_addresses_request = addresses[j:j + 25]
response = send_request(origin_addresses, dest_addresses_request, API_key)
responses['origin_addresses'] = response['origin_addresses']
for destination_address in response['destination_addresses']:
responses['destination_addresses'].append(destination_address)
for row in response['rows']:
if len(responses['rows']) == 0:
responses['rows'].append(row)
else:
for element in row['elements']:
responses['rows'][0]['elements'].append(element)
distance_matrix += build_distance_matrix(responses)
responses['origin_addresses'].clear()
responses['destination_addresses'].clear()
responses['rows'].clear()
return distance_matrix
def send_request(origin_addresses, dest_addresses, API_key):
""" Build and send request for the given origin and destination addresses."""
def build_address_str(addresses):
# Build a pipe-separated string of addresses
address_str = ''
for i in range(len(addresses) - 1):
address_str += addresses[i] + '|'
address_str += addresses[-1]
return address_str
request = 'https://maps.googleapis.com/maps/api/distancematrix/json?units=imperial'
origin_address_str = build_address_str(origin_addresses)
dest_address_str = build_address_str(dest_addresses)
request = request + '&origins=' + origin_address_str + '&destinations=' + \
dest_address_str + '&key=' + API_key
jsonResult = urllib.request.urlopen(request).read()
response = json.loads(jsonResult)
return response
def build_distance_matrix(response):
distance_matrix = []
for row in response['rows']:
row_list = [row['elements'][j]['duration']['value'] for j in range(len(row['elements']))]
distance_matrix.append(row_list)
return distance_matrix
def process_message(message_body):
print(f"processing message: {message_body}")
data = json.loads(message_body)
data_matrix = {}
data_matrix['problem_id'] = data['problemId']
data_matrix["addresses"] = []
data_matrix["pickups_deliveries"] = []
data_matrix["demands"] = []
data_matrix["jobOrderIDs"] = []
depot_address = str(data["depot"]["latitude"]) + "," + str(data["depot"]["longitude"])
data_matrix["jobOrderIDs"].append(0)
data_matrix["addresses"].append(depot_address)
hash_key = data["hashKey"]
for location in data["locationList"]:
pick_lat = location["PickupLatitude"]
pick_long = location["PickupLongitude"]
drop_lat = location["DropoffLatitude"]
drop_long = location["DropoffLongitude"]
jobOrderId = location["jobOrderID"]
demand = math.ceil(float(int(location["totalQuantity"]) / 20))
for i in range(0, demand):
data_matrix["addresses"].append(str(pick_lat) + ',' + str(pick_long))
data_matrix["addresses"].append(str(drop_lat) + ',' + str(drop_long))
data_matrix["jobOrderIDs"].append(str(jobOrderId))
data_matrix["jobOrderIDs"].append(str(jobOrderId))
data_matrix["demands"].append(0)
for i in range(1, len(data_matrix["addresses"]) - 1, 2):
data_matrix["pickups_deliveries"].append([i, i + 1])
data_matrix["demands"].append(1)
data_matrix["demands"].append(-1)
data_matrix["num_vehicles"] = int(len(data_matrix["addresses"]) / 2)
data_matrix["vehicle_capacities"] = []
for i in range(0, data_matrix["num_vehicles"]):
data_matrix["vehicle_capacities"].append(1)
data_matrix["depot"] = 0
data_matrix["API_key"] = ''
data_matrix["distance_matrix"] = create_distance_matrix(data_matrix)
# Create the routing index manager.
manager = pywrapcp.RoutingIndexManager(len(data_matrix['distance_matrix']),
data_matrix['num_vehicles'], data_matrix['depot'])
# Create Routing Model.
routing = pywrapcp.RoutingModel(manager)
# Define cost of each arc.
def distance_callback(from_index, to_index):
"""Returns the manhattan distance between the two nodes."""
# Convert from routing variable Index to distance matrix NodeIndex.
from_node = manager.IndexToNode(from_index)
to_node = manager.IndexToNode(to_index)
return data_matrix['distance_matrix'][from_node][to_node]*1000
transit_callback_index = routing.RegisterTransitCallback(distance_callback)
routing.SetArcCostEvaluatorOfAllVehicles(transit_callback_index)
# Add Distance constraint.
dimension_name = 'Duration'
routing.AddDimension(
transit_callback_index,
0, # no slack
28800*1000, # vehicle maximum travel hours
True, # start cumul to zero
dimension_name)
distance_dimension = routing.GetDimensionOrDie(dimension_name)
distance_dimension.SetGlobalSpanCostCoefficient(100)
def demand_callback(from_index):
"""Returns the demand of the node."""
# Convert from routing variable Index to demands NodeIndex.
from_node = manager.IndexToNode(from_index)
return data_matrix['demands'][from_node]
demand_callback_index = routing.RegisterUnaryTransitCallback(
demand_callback)
routing.AddDimensionWithVehicleCapacity(
demand_callback_index,
0, # null capacity slack
data_matrix['vehicle_capacities'], # vehicle maximum capacities
True, # start cumul to zero
'Capacity')
# Define Transportation Requests.
for request in data_matrix['pickups_deliveries']:
pickup_index = manager.NodeToIndex(request[0])
delivery_index = manager.NodeToIndex(request[1])
routing.AddPickupAndDelivery(pickup_index, delivery_index)
routing.solver().Add(
routing.VehicleVar(pickup_index) == routing.VehicleVar(
delivery_index))
routing.solver().Add(
distance_dimension.CumulVar(pickup_index) <=
distance_dimension.CumulVar(delivery_index))
# Setting first solution heuristic.
search_parameters = pywrapcp.DefaultRoutingSearchParameters()
search_parameters.local_search_metaheuristic = (
routing_enums_pb2.LocalSearchMetaheuristic.GUIDED_LOCAL_SEARCH)
search_parameters.time_limit.seconds = 1200
search_parameters.log_search = True
search_parameters.first_solution_strategy = (
routing_enums_pb2.FirstSolutionStrategy.AUTOMATIC)
search_parameters.use_full_propagation = True
# Solve the problem.
solution = routing.SolveWithParameters(search_parameters)
if solution:
solution_dict = {}
for vehicle_id in range(data_matrix['num_vehicles']):
index = routing.Start(vehicle_id)
plan_output = ''
route_distance = 0
route_load = 0
while not routing.IsEnd(index):
node_index = manager.IndexToNode(index)
plan_output += '{0},'.format(data_matrix['jobOrderIDs'][node_index])
previous_index = index
index = solution.Value(routing.NextVar(index))
plan_output += '{0},'.format(data_matrix['jobOrderIDs'][manager.IndexToNode(index)])
plan_output = plan_output[:-1]
plan_words = plan_output.split(",")
plan_output = ''
for i in range(len(plan_words)):
if (i % 2 == 0):
plan_output += plan_words[i] + ","
plan_output = plan_output[:-1]
plan_output += ",0"
if plan_output != 0 and plan_output != str(0) and plan_output != str('0,0'):
print(plan_output)
solution_dict[vehicle_id] = plan_output
# trucks_url = 'https://test-api.truckit.com/api/2/trucks'
trucks_url = 'https://test-api.truckit.com/api/2/job-orders/smart-dispatch/' + str(data_matrix['problem_id'])
head = build_auth_header(api_data["authToken"])
status = {}
ride_list = []
dummy_location_dict = {}
dummy_id_dict = {}
dummy_id_dict["id"] = 0
dummy_id_dict["name"] = ""
dummy_location_dict["location"] = dummy_id_dict
dummy_location_dict["timestamp"] = 0
ride_list.append(dummy_location_dict)
redisClient.hset(hash_key, "solution", json.dumps(solution_dict))
redisClient.hset(hash_key, "ride_list", json.dumps(ride_list))
json_data = {"status": "completed"}
api_response = requests.post(trucks_url, headers=head, json=json_data)
print_solution(data_matrix, manager, routing, solution)
def print_solution(data, manager, routing, solution):
"""Prints solution on console."""
print(f'Objective: {solution.ObjectiveValue()}')
total_distance = 0
total_load = 0
for vehicle_id in range(data['num_vehicles']):
index = routing.Start(vehicle_id)
plan_output = 'Route for vehicle {}:\n'.format(vehicle_id)
route_distance = 0
route_load = 0
while not routing.IsEnd(index):
node_index = manager.IndexToNode(index)
plan_output += ' {0} -> '.format(node_index)
previous_index = index
index = solution.Value(routing.NextVar(index))
try:
distance = data['distance_matrix'][previous_index][index]
route_distance += distance
except:
distance = distance
plan_output += ' {0}\n'.format(manager.IndexToNode(index))
plan_output += 'Time of the route: {} hours\n'.format(str(float(route_distance / (60 * 60))))
print(plan_output)
total_distance += route_distance
print('Total distance of all routes: {}m'.format(total_distance))
if __name__ == "__main__":
signal_handler = SignalHandler()
while not signal_handler.received_signal:
messages = queue.receive_messages(
MaxNumberOfMessages=1,
WaitTimeSeconds=1
)
for message in messages:
try:
process_message(message.body)
message.delete()
except Exception as e:
print(f"exception while processing message: {repr(e)}")
traceback.print_exc()
continue
message.delete()
IF anyone has any suggestions as to what the problem may be, your help is greatly appreciated.
I have 3 VGG: VGGA, VGGB and VGG*, trained with the following training function:
def train(nets, loaders, optimizer, criterion, epochs=20, dev=None, save_param=False, model_name="valerio"):
# try:
nets = [n.to(dev) for n in nets]
model_a = module_unwrap(nets[0], True)
model_b = module_unwrap(nets[1], True)
model_c = module_unwrap(nets[2], True)
reg_loss = nn.MSELoss()
criterion.to(dev)
reg_loss.to(dev)
# Initialize history
history_loss = {"train": [], "val": [], "test": []}
history_accuracy = {"train": [], "val": [], "test": []}
# Store the best val accuracy
best_val_accuracy = 0
# Process each epoch
for epoch in range(epochs):
# Initialize epoch variables
sum_loss = {"train": 0, "val": 0, "test": 0}
sum_accuracy = {"train": [0,0,0], "val": [0,0,0], "test": [0,0,0]}
progbar = None
# Process each split
for split in ["train", "val", "test"]:
if split == "train":
for n in nets:
n.train()
widgets = [
' [', pb.Timer(), '] ',
pb.Bar(),
' [', pb.ETA(), '] ', pb.Variable('ta','[Train Acc: {formatted_value}]')
]
progbar = pb.ProgressBar(max_value=len(loaders[split][0]),widgets=widgets,redirect_stdout=True)
else:
for n in nets:
n.eval()
# Process each batch
for j,((input_a, labels_a),(input_b, labels_b)) in enumerate(zip(loaders[split][0],loaders[split][1])):
input_a = input_a.to(dev)
input_b = input_b.to(dev)
labels_a = labels_a.long().to(dev)
labels_b = labels_b.long().to(dev)
#print(labels_a.shape)
#labels_a = labels_a.squeeze()
#labels_b = labels_b.squeeze()
#labels_a = labels_a.unsqueeze(1)
#labels_b = labels_b.unsqueeze(1)
#print(labels_a.shape)
#labels_a = labels_a.argmax(-1)
#labels_b = labels_b.argmax(-1)
inputs = torch.cat([input_a,input_b],axis=0)
labels = torch.cat([labels_a, labels_b])
#labels = labels.squeeze()
#print(labels.shape)
#labels = labels.argmax(-1)
# Reset gradients
optimizer.zero_grad()
# Compute output
features_a = nets[0](input_a)
features_b = nets[1](input_b)
features_c = nets[2](inputs)
pred_a = torch.squeeze(nets[3](features_a))
pred_b = torch.squeeze(nets[3](features_b))
pred_c = torch.squeeze(nets[3](features_c))
loss = criterion(pred_a, labels_a) + criterion(pred_b, labels_b) + criterion(pred_c, labels)
for n in model_a:
layer_a = model_a[n]
layer_b = model_b[n]
layer_c = model_c[n]
if (isinstance(layer_a,nn.Conv2d)):
loss += lambda_reg * reg_loss(combo_fn(layer_a.weight,layer_b.weight),layer_c.weight)
if (layer_a.bias is not None):
loss += lambda_reg * reg_loss(combo_fn(layer_a.bias, layer_b.bias), layer_c.bias)
# Update loss
sum_loss[split] += loss.item()
# Check parameter update
if split == "train":
# Compute gradients
loss.backward()
# Optimize
optimizer.step()
# Compute accuracy
#https://discuss.pytorch.org/t/bcewithlogitsloss-and-model-accuracy-calculation/59293/ 2
#pred_labels_a = (pred_a >= 0.0).long() # Binarize predictions to 0 and 1
#pred_labels_b = (pred_b >= 0.0).long() # Binarize predictions to 0 and 1
#pred_labels_c = (pred_c >= 0.0).long() # Binarize predictions to 0 and 1
#print(pred_a.shape)
_,pred_label_a = torch.max(pred_a, dim = 1)
pred_labels_a = (pred_label_a == labels_a).float()
_,pred_label_b = torch.max(pred_b, dim = 1)
pred_labels_b = (pred_label_b == labels_b).float()
_,pred_label_c = torch.max(pred_c, dim = 1)
pred_labels_c = (pred_label_c == labels).float()
batch_accuracy_a = pred_labels_a.sum().item() / len(labels_a)
batch_accuracy_b = pred_labels_b.sum().item() / len(labels_b)
batch_accuracy_c = pred_labels_c.sum().item() / len(labels)
# Update accuracy
sum_accuracy[split][0] += batch_accuracy_a
sum_accuracy[split][1] += batch_accuracy_b
sum_accuracy[split][2] += batch_accuracy_c
if (split=='train'):
progbar.update(j, ta=batch_accuracy_c)
if (progbar is not None):
progbar.finish()
# Compute epoch loss/accuracy
epoch_loss = {split: sum_loss[split] / len(loaders[split][0]) for split in ["train", "val", "test"]}
epoch_accuracy = {split: [sum_accuracy[split][i] / len(loaders[split][0]) for i in range(len(sum_accuracy[split])) ] for split in ["train", "val", "test"]}
# # Store params at the best validation accuracy
# if save_param and epoch_accuracy["val"] > best_val_accuracy:
# # torch.save(net.state_dict(), f"{net.__class__.__name__}_best_val.pth")
# torch.save(net.state_dict(), f"{model_name}_best_val.pth")
# best_val_accuracy = epoch_accuracy["val"]
print(f"Epoch {epoch + 1}:")
# Update history
for split in ["train", "val", "test"]:
history_loss[split].append(epoch_loss[split])
history_accuracy[split].append(epoch_accuracy[split])
# Print info
print(f"\t{split}\tLoss: {epoch_loss[split]:0.5}\tVGG 1:{epoch_accuracy[split][0]:0.5}"
f"\tVGG 2:{epoch_accuracy[split][1]:0.5}\tVGG *:{epoch_accuracy[split][2]:0.5}")
if save_param:
torch.save({'vgg_a':nets[0].state_dict(),'vgg_b':nets[1].state_dict(),'vgg_star':nets[2].state_dict(),'classifier':nets[3].state_dict()},f'{model_name}.pth')
For each epoch of training the result is this:
Then, I have a combined model which sums the weights of VGGA and VGGB:
DO = 'TEST'
if (DO=='TRAIN'):
train(nets, loaders, optimizer, criterion, epochs=50, dev=dev,save_param=True)
else:
state_dicts = torch.load('valerio.pth')
model1.load_state_dict(state_dicts['vgg_a']) #questi state_dict vengono dalla funzione di training
model2.load_state_dict(state_dicts['vgg_b'])
model3.load_state_dict(state_dicts['vgg_star'])
classifier.load_state_dict(state_dicts['classifier'])
test(model1,classifier,test_loader_all)
test(model2, classifier, test_loader_all)
test(model3, classifier, test_loader_all)
summed_state_dict = OrderedDict()
for key in state_dicts['vgg_star']:
if key.find('conv') >=0:
print(key)
summed_state_dict[key] = combo_fn(state_dicts['vgg_a'][key],state_dicts['vgg_b'][key])
else:
summed_state_dict[key] = state_dicts['vgg_star'][key]
model3.load_state_dict(summed_state_dict)
test(model3, classifier, test_loader_all)
where the test function is this:
def test(net,classifier, loader):
net.to(dev)
classifier.to(dev)
net.eval()
sum_accuracy = 0
# Process each batch
for j, (input, labels) in enumerate(loader):
input = input.to(dev)
labels = labels.float().to(dev)
features = net(input)
pred = torch.squeeze(classifier(features))
# https://discuss.pytorch.org/t/bcewithlogitsloss-and-model-accuracy-calculation/59293/ 2
#pred_labels = (pred >= 0.0).long() # Binarize predictions to 0 and 1
_,pred_label = torch.max(pred, dim = 1)
pred_labels = (pred_label == labels).float()
batch_accuracy = pred_labels.sum().item() / len(labels)
# Update accuracy
sum_accuracy += batch_accuracy
epoch_accuracy = sum_accuracy / len(loader)
print(f"Accuracy after sum: {epoch_accuracy:0.5}")
And the result of this aggregation is the following:
I want to modify my training function in order to print the same things of the first image, plus the accuracy of the aggregated model (the highlighted part in red of the second picture). So basically, for each epoch, accuracies of VGGA, VGGB, VGG* and combined VGG, print these accuracies and continue with the training. I tried to add this model combo but I failed, because I did not able to insert into each epoch, but only at the end of the training. I was trying to add in the training function, between print(f"Epoch {epoch + 1}:")and
# Update history
for split in ["train", "val", "test"]:
the code with the part of state_dict, but i am doing something wrong, i do not know what.
Can I reuse the code of the test, or I have to write new code?
Do you think i have to save the state_dict for each epoch, or i can do something else? Like model_c.parameters()=model_a.parameters()+model_b.parameters() (which does not work, already tried)
I solved, here is the solution of how I modified my training function:
def train(nets, loaders, optimizer, criterion, epochs=20, dev=None, save_param=False, model_name="valerio"):
# try:
nets = [n.to(dev) for n in nets]
model_a = module_unwrap(nets[0], True)
model_b = module_unwrap(nets[1], True)
model_c = module_unwrap(nets[2], True)
reg_loss = nn.MSELoss()
criterion.to(dev)
reg_loss.to(dev)
# Initialize history
history_loss = {"train": [], "val": [], "test": []}
history_accuracy = {"train": [], "val": [], "test": []}
history_test = 0
# Store the best val accuracy
best_val_accuracy = 0
# Process each epoch
for epoch in range(epochs):
# Initialize epoch variables
sum_loss = {"train": 0, "val": 0, "test": 0}
sum_accuracy = {"train": [0,0,0], "val": [0,0,0], "test": [0,0,0]}
progbar = None
# Process each split
for split in ["train", "val", "test"]:
if split == "train":
for n in nets:
n.train()
widgets = [
' [', pb.Timer(), '] ',
pb.Bar(),
' [', pb.ETA(), '] ', pb.Variable('ta','[Train Acc: {formatted_value}]')
]
progbar = pb.ProgressBar(max_value=len(loaders[split][0]),widgets=widgets,redirect_stdout=True)
else:
for n in nets:
n.eval()
# Process each batch
for j,((input_a, labels_a),(input_b, labels_b)) in enumerate(zip(loaders[split][0],loaders[split][1])):
input_a = input_a.to(dev)
input_b = input_b.to(dev)
labels_a = labels_a.long().to(dev)
labels_b = labels_b.long().to(dev)
#print(labels_a.shape)
#labels_a = labels_a.squeeze()
#labels_b = labels_b.squeeze()
#labels_a = labels_a.unsqueeze(1)
#labels_b = labels_b.unsqueeze(1)
#print(labels_a.shape)
#labels_a = labels_a.argmax(-1)
#labels_b = labels_b.argmax(-1)
inputs = torch.cat([input_a,input_b],axis=0)
labels = torch.cat([labels_a, labels_b])
#labels = labels.squeeze()
#print(labels.shape)
#labels = labels.argmax(-1)
# Reset gradients
optimizer.zero_grad()
# Compute output
features_a = nets[0](input_a)
features_b = nets[1](input_b)
features_c = nets[2](inputs)
pred_a = torch.squeeze(nets[3](features_a))
pred_b = torch.squeeze(nets[3](features_b))
pred_c = torch.squeeze(nets[3](features_c))
loss = criterion(pred_a, labels_a) + criterion(pred_b, labels_b) + criterion(pred_c, labels)
for n in model_a:
layer_a = model_a[n]
layer_b = model_b[n]
layer_c = model_c[n]
if (isinstance(layer_a,nn.Conv2d)):
loss += lambda_reg * reg_loss(combo_fn(layer_a.weight,layer_b.weight),layer_c.weight)
if (layer_a.bias is not None):
loss += lambda_reg * reg_loss(combo_fn(layer_a.bias, layer_b.bias), layer_c.bias)
# Update loss
sum_loss[split] += loss.item()
# Check parameter update
if split == "train":
# Compute gradients
loss.backward()
# Optimize
optimizer.step()
# Compute accuracy
#https://discuss.pytorch.org/t/bcewithlogitsloss-and-model-accuracy-calculation/59293/ 2
#pred_labels_a = (pred_a >= 0.0).long() # Binarize predictions to 0 and 1
#pred_labels_b = (pred_b >= 0.0).long() # Binarize predictions to 0 and 1
#pred_labels_c = (pred_c >= 0.0).long() # Binarize predictions to 0 and 1
#print(pred_a.shape)
_,pred_label_a = torch.max(pred_a, dim = 1)
pred_labels_a = (pred_label_a == labels_a).float()
_,pred_label_b = torch.max(pred_b, dim = 1)
pred_labels_b = (pred_label_b == labels_b).float()
_,pred_label_c = torch.max(pred_c, dim = 1)
pred_labels_c = (pred_label_c == labels).float()
batch_accuracy_a = pred_labels_a.sum().item() / len(labels_a)
batch_accuracy_b = pred_labels_b.sum().item() / len(labels_b)
batch_accuracy_c = pred_labels_c.sum().item() / len(labels)
# Update accuracy
sum_accuracy[split][0] += batch_accuracy_a
sum_accuracy[split][1] += batch_accuracy_b
sum_accuracy[split][2] += batch_accuracy_c
if (split=='train'):
progbar.update(j, ta=batch_accuracy_c)
if (progbar is not None):
progbar.finish()
# Compute epoch loss/accuracy
epoch_loss = {split: sum_loss[split] / len(loaders[split][0]) for split in ["train", "val", "test"]}
epoch_accuracy = {split: [sum_accuracy[split][i] / len(loaders[split][0]) for i in range(len(sum_accuracy[split])) ] for split in ["train", "val", "test"]}
# # Store params at the best validation accuracy
# if save_param and epoch_accuracy["val"] > best_val_accuracy:
# # torch.save(net.state_dict(), f"{net.__class__.__name__}_best_val.pth")
# torch.save(net.state_dict(), f"{model_name}_best_val.pth")
# best_val_accuracy = epoch_accuracy["val"]
print(f"Epoch {epoch + 1}:")
# Update history
for split in ["train", "val", "test"]:
history_loss[split].append(epoch_loss[split])
history_accuracy[split].append(epoch_accuracy[split])
# Print info
print(f"\t{split}\tLoss: {epoch_loss[split]:0.5}\tVGG 1:{epoch_accuracy[split][0]:0.5}"
f"\tVGG 2:{epoch_accuracy[split][1]:0.5}\tVGG *:{epoch_accuracy[split][2]:0.5}")
if save_param:
torch.save({'vgg_a':nets[0].state_dict(),'vgg_b':nets[1].state_dict(),'vgg_star':nets[2].state_dict(),'classifier':nets[3].state_dict()},f'{model_name}.pth')
test(nets[0], nets[3], test_loader_all)
test(nets[1], nets[3], test_loader_all)
test(nets[2], nets[3], test_loader_all)
summed_state_dict = OrderedDict()
for key in nets[2].state_dict():
if key.find('conv') >=0:
#print(key)
summed_state_dict[key] = combo_fn(nets[0].state_dict()[key],nets[1].state_dict()[key])
else:
summed_state_dict[key] = nets[2].state_dict()[key]
nets[2].load_state_dict(summed_state_dict)
test(nets[2], nets[3], test_loader_all)
The edited parts are the last rows.
I am struggling to understand why this function apparently fails in the Jupyter Notebook, but not in the IPython shell:
def present_value( r, n, fv = None, pmt = None ):
'''
Function to compute the Present Value based on interest rate and
a given future value.
Arguments accepted
------------------
* r = interest rate,
which should be given in its original percentage, eg.
5% instead of 0.05
* n = number of periods for which the cash flow,
either as annuity or single flow from one present value
* fv = future value in dollars,
if problem is annuity based, leave this empty
* pmt = each annuity payment in dollars,
if problem is single cash flow based, leave this empty
'''
original_args = [r, n, fv, pmt]
dec_args = [Decimal( arg ) if arg != None
else arg
for arg in original_args
]
if dec_args[3] == None:
return dec_args[2] / ( ( 1 + ( dec_args[0] / 100 ) )**dec_args[1] )
elif dec_args[2] == None:
# annuity_length = range( 1, dec_args[1] + 1 )
# Not allowed to add a Decimal object
# with an integer and to use it
# in the range() function,
# so we dereference the integer from original_args
annuity_length = range( 1, original_args[1] + 1 )
# Apply discounting to each annuity payment made
# according to number of years left till end
all_compounded_pmt = [dec_args[3] * ( 1 / ( ( 1 + dec_args[0] / 100 ) ** time_left ) ) \
for time_left in annuity_length
]
return sum( all_compounded_pmt )
When I imported the module that this function resides in, named functions.py, using from functions import *, and then executed present_value(r=7, n=35, pmt = 11000), I got the error:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-93-c1cc587f7e27> in <module>()
----> 1 present_value(r=7, n=35, pmt = 11000)
/path_to_file/functions.py in present_value(r, n, fv, pmt)
73 if dec_args[3] == None:
74 return dec_args[2]/((1 + (dec_args[0]/100))**dec_args[1])
---> 75
76 elif dec_args[2] == None:
77 # annuity_length = range(1, dec_args[1]+1)
TypeError: 'decimal.Decimal' object cannot be interpreted as an integer
but in the IPython shell, evaluating this function it works perfectly fine:
In [42]: functions.present_value(r=7, n=35, pmt = 11000)
Out[42]: Decimal('142424.39530474029537')
Can anyone please help me with this really confusing and obscure issue?
>>> def elp(min):
... when = datetime.now() + timedelta(minutes=min)
... print when
... r = add.apply_async(args=[500,500],eta=when)
... start = time.time()
... r.get()
... end = time.time()
... elapsed = end-start
... print elapsed
...
>>> elp(10)
2014-11-08 04:38:01.745000
1.00200009346
Where as when using countdown
>>> def elp_countdown(min):
... r = add.apply_async(args=[500,500],countdown=(min*60))
... start = time.time()
... r.get()
... end = time.time()
... elapsed = end-start
... print elapsed
...
>>> elp_countdown(0.5)
30.1380000114
Why does the task gets executed faster than its suppose when using eta?
my add task is as follows,
#task()
def add(x, y):
return x + y
Celery verison: 3.1.16 (Cipater)
So after some research and lot's of reading, it turns out that I needed to pass a utc datetime to celery, since it's time zone is configured by default to be in UTC.
In case anyone is wondering, changing to this:
when = datetime.utcnow() + timedelta(minutes=min)
instead of
when = datetime.now() + timedelta(minutes=min)
will make it work as it should.