I'm writing a function that computes the sparsity of the weight matrices of the following fully connected network:
class FCN(nn.Module):
def __init__(self):
super(FCN, self).__init__()
self.fc1 = nn.Linear(input_dim, hidden_dim)
self.relu1 = nn.ReLU()
self.fc2 = nn.Linear(hidden_dim, hidden_dim)
self.relu2 = nn.ReLU()
self.fc3 = nn.Linear(hidden_dim, hidden_dim)
self.relu3 = nn.ReLU()
self.fc4 = nn.Linear(hidden_dim, output_dim)
def forward(self, x):
out = self.fc1(x)
out = self.relu1(out)
out = self.fc2(out)
out = self.relu2(out)
out = self.fc3(out)
out = self.relu3(out)
out = self.fc4(out)
return out
The function I have written is the following:
def print_layer_sparsity(model):
for name,module in model.named_modules():
if 'fc' in name:
zeros = 100. * float(torch.sum(model.name.weight == 0))
tot = float(model.name.weight.nelement())
print("Sparsity in {}.weight: {:.2f}%".format(name, zeros/tot))
But it gives me the following error:
torch.nn.modules.module.ModuleAttributeError: 'FCN' object has no attribute 'name'
It works fine when I manually enter the name of the layers (e.g.,
(model.fc1.weight == 0)
(model.fc2.weight == 0)
(model.fc3.weight == 0) ....
but I'd like to make it independent from the network. In other words, I'd like to adapt my function in a way that, given any sparse network, it prints the sparsity of every layer. Any suggestions?
Thanks!!
Try:
getattr(model, name).weight
In place of
model.name.weight
Your print_layer_sparsity function becomes:
def print_layer_sparsity(model):
for name,module in model.named_modules():
if 'fc' in name:
zeros = 100. * float(torch.sum(getattr(model, name).weight == 0))
tot = float(getattr(model, name).weight.nelement())
print("Sparsity in {}.weight: {:.2f}%".format(name, zeros/tot))
You can't do model.name because name is a str. The in-built getattr function allows you to get the member variables / attributes of an object using its name as a string.
For more information, checkout this answer.
I want to automatically rename repeated columns of a df. For example:
df
Out[4]: DataFrame[norep1: string, num1: string, num1: bigint, norep2: bigint, num1: bigint, norep3: bigint]
Apply some function to end with a df like:
f_rename_repcol(df)
Out[4]: DataFrame[norep1: string, num1_1: string, num1_2: bigint, norep2: bigint, num1_3: bigint, norep3: bigint]
I've already create my own function, and works, but I'm sure there is a shorter and better way of doing it:
def f_df_col_renombra_rep(df):
from collections import Counter
from itertools import chain
import pandas as pd
columnas_original = np.array(df.columns)
d1 = Counter(df.columns)
i_corrige = [a>1 for a in dict(d1.items()).values()]
var_corrige = np.array(dict(d1.items()).keys())[i_corrige]
var_corrige_2 = [a for a in columnas_original if a in var_corrige]
columnas_nuevas = []
for var in var_corrige:
aux_corr = [a for a in var_corrige_2 if a in var]
i=0
columnas_nuevas_aux=[]
for valor in aux_corr:
i+=1
nombre_nuevo = valor +"_"+ str(i)
columnas_nuevas_aux.append(nombre_nuevo)
columnas_nuevas.append(columnas_nuevas_aux)
columnas_nuevas=list(chain.from_iterable(columnas_nuevas))
indice_cambio = pd.Series(columnas_original).isin(var_corrige)
i = 0
j = 0
colsalida = [None]*len(df.columns)
for col in df.columns:
if indice_cambio[i] == True:
colsalida[i] = columnas_nuevas[j]
j += 1
else:
colsalida[i] = col
# no cambio el nombre
i += 1
df_out = df.toDF(*(colsalida))
return df_out
You can modify the renaming function here to suit your need, but broadly I find this as the best way to rename all the duplicates columns
old_col=df.schema.names
running_list=[]
new_col=[]
i=0
for column in old_col:
if(column in running_list):
new_col.append(column+"_"+str(i))
i=i+1
else:
new_col.append(column)
running_list.append(column)
print(new_col)
This the conversion I do, the suffix assigned to the duplicate columns is not that of difference until the name(prefix) remains the same & I can save the file.
To update the columns you can simply run:
df=df.toDF(*new_col)
This should update the column names and remove all the duplicates
If you want to keep the numbering as _1,_2,_3:
You can use a dictionary and try and except block,
dict={}
for column in old_col:
try:
i=dict[column]+1
new_col.append(column+"_"+str(i))
dict[column]=i
except:
dict[column]=1
new_col.append(column+"_"+str(1)
print(new_col)
the easy way I am doing it is:
def col_duplicates(self):
'''rename dataframe with dups'''
columnas = self.columns.copy()
for i in range(len(columnas)-1):
for j in range(i+1, len(columnas), 1):
if columnas[i] == columnas[j]:
columnas[j] = columnas[i] + '_dup_' + str(j) # this line controls how to rename
return self.toDF(*columnas)
use as:
new_df_without_duplicates = col_duplicates(df_with_duplicates)
I'm trying to solve a car problem.
first, I have an original code of car problem:
# Ampl Car Example
#
# Shows how to convert a minimize final time optimal control problem
# to a format pyomo.dae can handle by removing the time scaling from
# the ContinuousSet.
#
# min tf
# dxdt = 0
# dvdt = a-R*v^2
# x(0)=0; x(tf)=L
# v(0)=0; v(tf)=0
# -3<=a<=1
from pyomo.environ import *
from pyomo.dae import *
m = ConcreteModel()
m.R = Param(initialize=0.001) # Friction factor
m.L = Param(initialize=100.0) # Final position
m.tau = ContinuousSet(initialize=[0.0, 0.80, 1.0]) # Unscaled time
m.time = Var(m.tau) # Scaled time
m.tf = Var()
m.x = Var(m.tau,bounds=(0,None))
m.v = Var(m.tau,bounds=(0,None))
m.a = Var(m.tau, bounds=(-3.0,1.0),initialize=0)
m.dtime = DerivativeVar(m.time)
m.dx = DerivativeVar(m.x)
m.dv = DerivativeVar(m.v)
m.obj = Objective(expr=m.tf)
def _ode1(m,i):
if i == 0 :
return Constraint.Skip
return m.dx[i] == m.tf * m.v[i]
m.ode1 = Constraint(m.tau, rule=_ode1)
def _ode2(m,i):
if i == 0 :
return Constraint.Skip
return m.dv[i] == m.tf*(m.a[i] - m.R*m.v[i]**2)
m.ode2 = Constraint(m.tau, rule=_ode2)
def _ode3(m,i):
if i == 0:
return Constraint.Skip
return m.dtime[i] == m.tf
m.ode3 = Constraint(m.tau, rule=_ode3)
def _init(m):
yield m.x[0] == 0
yield m.x[1] == m.L
yield m.v[0] == 0
yield m.v[1] == 0
yield m.time[0] == 0
m.initcon = ConstraintList(rule=_init)
discretizer = TransformationFactory('dae.collocation')
discretizer.apply_to(m,ncp=1, scheme='LAGRANGE-RADAU')
solver = SolverFactory('ipopt')
solver.solve(m, tee=True)
print("final time = %6.2f" %(value(m.tf)))
Now, I want to use class to express a car,then I could instantiate two cars.
So I write like this:
from pyomo.environ import *
from pyomo.dae import *
m = ConcreteModel()
class Car():
def __init__(self,friction):
self.friction = friction
self.R = Param(initialize = self.friction) # Friction factor
self.tau = ContinuousSet(bounds=(0, 1)) # Unscaled time
self.time = Var(self.tau) # Scaled time
self.tf = Var()
self.x = Var(self.tau, bounds=(0, None), initialize=0)
self.v = Var(self.tau, bounds=(0, None))
self.a = Var(self.tau, bounds=(-3.0, 1.0), initialize=0)
self.dtime = DerivativeVar(self.time)
self.dx = DerivativeVar(self.x)
self.dv = DerivativeVar(self.v)
def _ode1(m, i):
if i == 0:
return Constraint.Skip
return self.dx[i] == m.tf * self.v[i]
self.ode1 = Constraint(self.tau, rule=_ode1)
def _ode2(m, i):
if i == 0:
return Constraint.Skip
return self.dv[i] == m.tf * (self.a[i] - self.R * self.v[i] ** 2)
self.ode2 = Constraint(self.tau, rule=_ode2)
def _ode3(m, i):
if i == 0:
return Constraint.Skip
return self.dtime[i] == m.tf
self.ode3 = Constraint(self.tau, rule=_ode3)
m.car1 = Car(0.001)
m.obj = Objective(expr=m.car1.tf)
def _init(m):
yield m.car1.x[0] == 0
yield m.car1.x[1] == 100
yield m.car1.v[0] == 0
yield m.car1.v[1] == 0
yield m.car1.time[0] == 0
m.car1.initcon = ConstraintList(rule=_init)
discretizer = TransformationFactory('dae.finite_difference')
discretizer.apply_to(m, nfe=10, scheme='BACKWARD')
solver = SolverFactory('ipopt')
solver.solve(m, tee=True)
print("final time = %6.2f" % (value(m.car1.tf)))
However, I get this:
Traceback (most recent call last):
File "D:/pyo/pyomoceshi/ceshi3/car/classcar3.py", line 79, in <module>
solver.solve(m, tee=True)
File "D:\python\m\lib\site-packages\pyomo\opt\base\solvers.py", line 582, in solve
self._presolve(*args, **kwds)
File "D:\python\m\lib\site-packages\pyomo\opt\solver\shellcmd.py", line 196, in _presolve
OptSolver._presolve(self, *args, **kwds)
File "D:\python\m\lib\site-packages\pyomo\opt\base\solvers.py", line 661, in _presolve
**kwds)
File "D:\python\m\lib\site-packages\pyomo\opt\base\solvers.py", line 729, in _convert_problem
**kwds)
File "D:\python\m\lib\site-packages\pyomo\opt\base\convert.py", line 110, in convert_problem
problem_files, symbol_map = converter.apply(*tmp, **tmpkw)
File "D:\python\m\lib\site-packages\pyomo\solvers\plugins\converter\model.py", line 164, in apply
io_options=io_options)
File "D:\python\m\lib\site-packages\pyomo\core\base\block.py", line 1646, in write
io_options)
File "D:\python\m\lib\site-packages\pyomo\repn\plugins\ampl\ampl_.py", line 357, in __call__
include_all_variable_bounds=include_all_variable_bounds)
File "D:\python\m\lib\site-packages\pyomo\repn\plugins\ampl\ampl_.py", line 783, in _print_model_NL
list(self_varID_map[id(var)] for var in ampl_repn._linear_vars),
File "D:\python\m\lib\site-packages\pyomo\repn\plugins\ampl\ampl_.py", line 783, in <genexpr>
list(self_varID_map[id(var)] for var in ampl_repn._linear_vars),
KeyError: 68767416L
I want to know how to solve it or use other ways.
Below is a working version of your script. I changed things so that instead of a Car class there is a Car function that returns a Pyomo Block representing the car. By having a Car class you were essentially trying to create a subclass of Block and running into several subtle challenges that go along with that. You can see the blog post here for more information. The second change I made was in your declaration of the initial conditions, I changed the name of the ConstraintList from m.car1.initcon to m.car1_initcon. The difference is whether you want the ConstraintList to live on the car1 Block or the model. In your code, the 'dot' in the name meant you were trying to put it on the car1 Block but the constraints yielded in the rule were relative to the model. I changed the name to resolve this inconsistency.
from pyomo.environ import *
from pyomo.dae import *
m = ConcreteModel()
def Car(model, friction):
def construct_car_block(b):
b.R = Param(initialize = friction) # Friction factor
b.tau = ContinuousSet(bounds=(0, 1)) # Unscaled time
b.time = Var(b.tau) # Scaled time
b.tf = Var()
b.x = Var(b.tau, bounds=(0, None), initialize=0)
b.v = Var(b.tau, bounds=(0, None))
b.a = Var(b.tau, bounds=(-3.0, 1.0), initialize=0)
b.dtime = DerivativeVar(b.time)
b.dx = DerivativeVar(b.x)
b.dv = DerivativeVar(b.v)
def _ode1(b, i):
if i == 0:
return Constraint.Skip
return b.dx[i] == b.tf * b.v[i]
b.ode1 = Constraint(b.tau, rule=_ode1)
def _ode2(b, i):
if i == 0:
return Constraint.Skip
return b.dv[i] == b.tf * (b.a[i] - b.R * b.v[i] ** 2)
b.ode2 = Constraint(b.tau, rule=_ode2)
def _ode3(m, i):
if i == 0:
return Constraint.Skip
return b.dtime[i] == b.tf
b.ode3 = Constraint(b.tau, rule=_ode3)
return Block(rule=construct_car_block)
m.car1 = Car(m, friction=0.001)
m.obj = Objective(expr=m.car1.tf)
def _init(m):
yield m.car1.x[0] == 0
yield m.car1.x[1] == 100
yield m.car1.v[0] == 0
yield m.car1.v[1] == 0
yield m.car1.time[0] == 0
m.car1_initcon = ConstraintList(rule=_init)
discretizer = TransformationFactory('dae.finite_difference')
discretizer.apply_to(m, nfe=15, scheme='BACKWARD')
solver = SolverFactory('ipopt')
solver.solve(m, tee=True)
print("final time = %6.2f" % (value(m.car1.tf)))
I have a model that has been trained on CIFAR-10, but I don't realise how can I make a prediction in pycaffe.
I got an image from lmdb but I don't know how to load it in a net and get a predicted class.
My code:
net = caffe.Net('acc81/model.prototxt',
'acc81/cifar10_full_iter_70000.caffemodel.h5',
caffe.TEST)
lmdb_env = lmdb.open('cifar10_test_lmdb/')
lmdb_txn = lmdb_env.begin()
lmdb_cursor = lmdb_txn.cursor()
for key, value in lmdb_cursor:
datum = caffe.proto.caffe_pb2.Datum()
datum.ParseFromString(value)
image = caffe.io.datum_to_array(datum)
image = image.astype(np.uint8)
# What's next with the image variable?
# If i try:
# out = net.forward_all(data=np.asarray([image]))
# I get Exception: Input blob arguments do not match net inputs.
print("Image class is " + label)
Use this python script
# Run the script with anaconda-python
# $ /home/<path to anaconda directory>/anaconda/bin/python LmdbClassification.py
import sys
import numpy as np
import lmdb
import caffe
from collections import defaultdict
caffe.set_mode_gpu()
# Modify the paths given below
deploy_prototxt_file_path = '/home/<username>/caffe/examples/cifar10/cifar10_deploy.prototxt' # Network definition file
caffe_model_file_path = '/home/<username>/caffe/examples/cifar10/cifar10_iter_5000.caffemodel' # Trained Caffe model file
test_lmdb_path = '/home/<username>/caffe/examples/cifar10/cifar10_test_lmdb/' # Test LMDB database path
mean_file_binaryproto = '/home/<username>/caffe/examples/cifar10/mean.binaryproto' # Mean image file
# Extract mean from the mean image file
mean_blobproto_new = caffe.proto.caffe_pb2.BlobProto()
f = open(mean_file_binaryproto, 'rb')
mean_blobproto_new.ParseFromString(f.read())
mean_image = caffe.io.blobproto_to_array(mean_blobproto_new)
f.close()
# CNN reconstruction and loading the trained weights
net = caffe.Net(deploy_prototxt_file_path, caffe_model_file_path, caffe.TEST)
count = 0
correct = 0
matrix = defaultdict(int) # (real,pred) -> int
labels_set = set()
lmdb_env = lmdb.open(test_lmdb_path)
lmdb_txn = lmdb_env.begin()
lmdb_cursor = lmdb_txn.cursor()
for key, value in lmdb_cursor:
datum = caffe.proto.caffe_pb2.Datum()
datum.ParseFromString(value)
label = int(datum.label)
image = caffe.io.datum_to_array(datum)
image = image.astype(np.uint8)
out = net.forward_all(data=np.asarray([image]) - mean_image)
plabel = int(out['prob'][0].argmax(axis=0))
count += 1
iscorrect = label == plabel
correct += (1 if iscorrect else 0)
matrix[(label, plabel)] += 1
labels_set.update([label, plabel])
if not iscorrect:
print("\rError: key = %s, expected %i but predicted %i" % (key, label, plabel))
sys.stdout.write("\rAccuracy: %.1f%%" % (100.*correct/count))
sys.stdout.flush()
print("\n" + str(correct) + " out of " + str(count) + " were classified correctly")
print ""
print "Confusion matrix:"
print "(r , p) | count"
for l in labels_set:
for pl in labels_set:
print "(%i , %i) | %i" % (l, pl, matrix[(l,pl)])
I have 2 models:
class Restaurant(models.Model):
serves_hot_dogs = models.BooleanField(default=False)
serves_pizza = models.BooleanField(default=False)
def __str__(self): # __unicode__ on Python 2
return "%s the restaurant" % self.place.name
class Waiter(models.Model):
restaurant = models.ForeignKey(Restaurant, on_delete=models.CASCADE)
name = models.CharField(max_length=50)
def __str__(self): # __unicode__ on Python 2
return "%s the waiter at %s" % (self.name, self.restaurant)
if I want to get all waiters and restaurants with a single query using join I write:
w = Waiter.objects.select_related().all()
or
w = Waiter.objects.select_related().get(pk=1)
But how could get a restaurant with all waiters? I tried
r = Restaurant.objects.select_related().get(pk=1)
but it didn't work.
I do not want to do this with 2 db interogations like:
r = Restaurant.objects.get(pk=1)
w = r.waiter_set.all()
It is possible to get all informations with a single query?
Thank you