SciPy opitimize 'ValueError: setting an array element with a sequence.' ARIMA models - scipy

I know that the issue 'ValueError: setting an array element with a sequence.' is normally because the function being optimized is a vector and not a scalar, anyhow my ARMA model below still gets this issue. Here is the code
def loghood(parm,endog,exog,p,q):
arparams,maparams,exogparams,bias = parm
armapredicts=np.zeros(endog.shape[0])
bias=0
res=abs(endog - np.mean(endog))
if p==0:
for i in range(1,endog.shape[0]-p):
armapredicts[i] = np.array([[res[i-f+q]] for f in range(0,q)]).dot(maparams.T) + exog.iloc[i,:].dot(exogparams.T) + bias
if q==0:
for i in range(1,endog.shape[0]-q):
armapredicts[i] = np.array([ [endog[i-f+p]] for f in range(0,p)]).T.dot(arparams) + exog.iloc[i,:].T.dot(exogparams) + bias
else:
for i in range(1,endog.shape[0]-2):
armapredicts[i] = np.array([ [endog[i-f+p]] for f in range(0,p)]).T.dot(arparams.reshape(-1,1)) + np.array([[res[i-f+q]] for f in range(0,q)]).T.dot(maparams.reshape(-1,1)) + exog.iloc[i,:].T.dot(exogparams.reshape(-1,1)) + bias
print(np.array([ [endog[i-f+p]] for f in range(0,p)]).T.shape )
print(maparams.reshape(-1,1).shape)
liklihood=1/((2*np.pi*armapredicts)**(1/2))*np.exp(-res**2/(2*armapredicts**2))
print(liklihood.shape)
log_hood=np.sum(np.log(liklihood.values))
print(log_hood)
return log_hood
x0=[np.ones(2),np.ones(2),np.ones(returnsant.shape[1]-1),1]
x0=np.array(x0,dtype=object).flatten()
res = spop.minimize(loghood,x0 ,args=(returnsant['Hedge Fund'],returnsant.drop('Hedge Fund',axis= 1),2,2), method='Nelder-mead')
print(res)
I know that likelihood is a 1-d vector and the log_hood is certainly scalar after np.sum, so how is this error occurring?? Thank you for your time.
EDIT:forgot to include the full error message
TypeError Traceback (most recent call last)
TypeError: only size-1 arrays can be converted to Python scalars
The above exception was the direct cause of the following exception:
ValueError Traceback (most recent call last)
/var/folders/7f/gmpqnwqx0lb4nrsz2vqhvgn40000gp/T/ipykernel_30671/32993526.py in <module>
1 x0=[np.ones(2),np.ones(2),np.ones(returnsant.shape[1]-1),1]
2 #x0=np.array(x0,dtype=object).flatten()
----> 3 res = spop.minimize(loghood,x0 ,args=(returnsant['Hedge Fund'],returnsant.drop('Hedge Fund',axis= 1),2,2), method='Nelder-mead')
4 print(res)
~/opt/anaconda3/lib/python3.9/site-packages/scipy/optimize/_minimize.py in minimize(fun, x0, args, method, jac, hess, hessp, bounds, constraints, tol, callback, options)
609
610 if meth == 'nelder-mead':
--> 611 return _minimize_neldermead(fun, x0, args, callback, bounds=bounds,
612 **options)
613 elif meth == 'powell':
~/opt/anaconda3/lib/python3.9/site-packages/scipy/optimize/optimize.py in _minimize_neldermead(func, x0, args, callback, maxiter, maxfev, disp, return_all, initial_simplex, xatol, fatol, adaptive, bounds, **unknown_options)
687 zdelt = 0.00025
688
--> 689 x0 = asfarray(x0).flatten()
690
691 if bounds is not None:
~/opt/anaconda3/lib/python3.9/site-packages/numpy/core/overrides.py in asfarray(*args, **kwargs)
~/opt/anaconda3/lib/python3.9/site-packages/numpy/lib/type_check.py in asfarray(a, dtype)
112 if not _nx.issubdtype(dtype, _nx.inexact):
113 dtype = _nx.float_
--> 114 return asarray(a, dtype=dtype)
115
116
ValueError: setting an array element with a sequence.
FINAL EDIT:
I resolved the issue by simply creating a big array x0 of all my parameters
x0=np.zeros(5+returnsant.shape[1]-1)
res = spop.minimize(loghood,x0,args=(returnsant['Hedge Fund'],returnsant.drop('Hedge Fund',axis= 1),0,2), method='Nelder-mead')
print(res)
now I declare my subparameters in loghood
def loghood(parms,endog,exog,p,q):
arparams,maparams,exogparams,bias = parms[0:p],parms[p:p+q],parms[p+q:p+q+exog.shape[1]],parms[p+q+exog.shape[1]:-1]
armapredicts=np.zeros(endog.shape[0])
bias=0
res=abs(endog - np.mean(endog))
if p==0:
for i in range(1,endog.shape[0]-q):
armapredicts[i] = np.array([[res[i-f+q]] for f in range(0,q)]).T.dot(maparams) + exog.iloc[i,:].T.dot(exogparams) + bias
if q==0:
for i in range(1,endog.shape[0]-p):
armapredicts[i] = np.array([ [endog[i-f+p]] for f in range(0,p)]).T.dot(arparams) + exog.iloc[i,:].T.dot(exogparams) + bias
else:
for i in range(1,endog.shape[0]-2):
armapredicts[i] = np.array([ [endog[i-f+p]] for f in range(0,p)]).T.dot(arparams.reshape(-1,1)) + np.array([[res[i-f+q]] for f in range(0,q)]).T.dot(maparams.reshape(-1,1)) + exog.iloc[i,:].T.dot(exogparams.reshape(-1,1)) + bias
liklihood=1/((2*np.pi*armapredicts)**(1/2))*np.exp(-res**2/(2*armapredicts**2))
log_hood=np.sum(-np.log(liklihood.squeeze()))
return log_hood

Related

IndexError target is out of bounds

I'm working on a custom dataset of images and using a Neural Net to classify them.
The data set is about 6000 images of 58 classes. But on training I keep getting a "target is out of bounds" error.
I've double checked the number of classes and image size but still get the same error.
#hyperprams
learning_rate = 5e-4
#3 for RGB values
in_channel = 3
#classes from data set
num_classes = 58
# arbitray choice
batch_size = 32
#total number of epochs used to train the model
epochs = 3
traffic_dataset = TrafficSigns(csv_file='annotations.csv',
root_directory='/Users/*****/Desktop/images/',
transform = transforms.ToTensor())
train_size = int(0.8 * len(traffic_dataset))
test_size = len(traffic_dataset) - train_size
train, test = torch.utils.data.random_split(traffic_dataset,
[train_size, test_size])
train_loader = torch.utils.data.DataLoader(train,
batch_size= batch_size,
shuffle= True,
num_workers= 4)
test_loader = torch.utils.data.DataLoader(test,
batch_size = batch_size,
shuffle= True,
num_workers= 4)
#Create a fully connected nn
class Net(nn.Module):
#use the constructor w/ arguments size of data and number of classes
def __init__(self,
input_size,
num_classes):
super(Net, self).__init__()
self.fc1 = nn.Linear(input_size, 60)
self.fc2 = nn.Linear(60, num_classes)
#define your forward step function with relu as the non-linear function of the weights
#x will be the datapassed to the model
def forward(self, x):
x=f.relu(self.fc1(x))
x = self.fc2(x)
return x
#sanity check
test = Net(2028, num_classes)
x = torch.randn(24, 2028)
print(test(x).shape)
#instantiate the class object of NN
net = Net(2028, num_classes)
criterion = nn.CrossEntropyLoss()
nn_optimizer = optim.Adam(net.parameters(),
lr = learning_rate)
#train on multiple epochs using the criterion and gradient decent algorthim estabilished above
for epoch in range(1):
for i, (data, target) in enumerate(tqdm.tqdm(train_loader)):
data = data.reshape(data.shape[0], -1)
#forward
outputs = net(data)
loss = criterion(outputs, target)
#backward propigation
nn_optimizer.zero_grad()
loss.backward()
#gradiant decent choosen
nn_optimizer.step()
Im also using a custom dataset class to import the images and labels.
My first thought was that the class is not iterating over the CSV and images correctly but I can't seem to find where they might be not matching up.
class TrafficSigns(Dataset):
#constructure will need csv file of labels images and the transform function defined above
def __init__(self,
csv_file,
root_directory,
transform = None):
self.labels = pd.read_csv(csv_file)
self.root_directory = root_directory
self.transform = transform
#returns the length
def __len__(self):
return len(self.labels)
#get data index by indes
def __getitem__(self, i):
image_path = os.path.join(self.root_directory, self.labels.iloc[i,0])
image = io.imread(image_path)
y_label = torch.tensor(int(self.labels.iloc[i, 1]))
#if statement needed since transform can be set to None
if self.transform:
image = self.transform(image)
return (image, y_label)
Any help would be awesome, thank you.
Here is the full stacktrace error that's getting thrown.
IndexError Traceback (most recent call last)
/var/folders/t_/rcfcs8g56jn7trwnsvmdyh_r0000gn/T/ipykernel_34551/1839343274.py in <module>
11 #forward
12 outputs = net(data)
---> 13 loss = criterion(outputs, target)
14 #backward propigation
15 nn_optimizer.zero_grad()
~/Library/Python/3.8/lib/python/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
1100 if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
1101 or _global_forward_hooks or _global_forward_pre_hooks):
-> 1102 return forward_call(*input, **kwargs)
1103 # Do not call functions when jit is used
1104 full_backward_hooks, non_full_backward_hooks = [], []
~/Library/Python/3.8/lib/python/site-packages/torch/nn/modules/loss.py in forward(self, input, target)
1148
1149 def forward(self, input: Tensor, target: Tensor) -> Tensor:
-> 1150 return F.cross_entropy(input, target, weight=self.weight,
1151 ignore_index=self.ignore_index, reduction=self.reduction,
1152 label_smoothing=self.label_smoothing)
~/Library/Python/3.8/lib/python/site-packages/torch/nn/functional.py in cross_entropy(input, target, weight, size_average, ignore_index, reduce, reduction, label_smoothing)
2844 if size_average is not None or reduce is not None:
2845 reduction = _Reduction.legacy_get_string(size_average, reduce)
-> 2846 return torch._C._nn.cross_entropy_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index, label_smoothing)
2847
2848
IndexError: Target 125 is out of bounds.
I came across same issue where I used sequential model (LSTM) for next sequence prediction. I check data loader where labels contained -1 because of which cross entropy loss throwing exception. here is my sequence chunks where model found -1 sequence as label in data loader:
Solved please check your null rows and remove those or set accordingly.

getting __init__() got an unexpected keyword argument 'outputCols' when doing the oneHotEncoder

stages = []
for categoricalCol in categoricalColumns:
stringIndexer = StringIndexer(
inputCol=categoricalCol, outputCol=categoricalCol + "Index"
)
encoder = OneHotEncoder(
inputCols=[stringIndexer.getOutputCol()],
outputCols=[categoricalCol + "classVec"],
)
stages += [stringIndexer, encoder]
label_stringIdx = StringIndexer(inputCol="Status", outputCol="label")
stages += [label_stringIdx]
assemblerInputs = [c + "classVsc" for c in categoricalColumns] + numericColumns
assembler = VectorAssembler(inputCols=assemblerInputs, outputCol="features")
stages += [assembler]
> TypeError
> Traceback (most recent call last) <ipython-input-18-7156eaeec61b> in <module>
> 2 for categoricalCol in categoricalColumns:
> 3 stringIndexer = StringIndexer(inputCol = categoricalCol, outputCol = categoricalCol + 'Index')
> ----> 4 encoder = OneHotEncoder(inputCols=[stringIndexer.getOutputCol()], outputCols=[categoricalCol + "classVec"])
> 5 stages += [stringIndexer, encoder]
> 6 label_stringIdx = StringIndexer(inputCol = 'Status', outputCol = 'label')
>
> /usr/local/spark/python/pyspark/__init__.py in wrapper(self, *args, **kwargs)
> 102 raise TypeError("Method %s forces keyword arguments." % func.__name__)
> 103 self._input_kwargs = kwargs
> --> 104 return func(self, **kwargs)
> 105 return wrapper
> 106
>
> TypeError: __init__() got an unexpected keyword argument 'outputCols'
Check the documentation:
https://spark.apache.org/docs/2.2.2/api/python/pyspark.ml.html?highlight=onehotencoder#pyspark.ml.feature.OneHotEncoder
The param is outputCol

NameError: name 'pbc' is not defined

Hello I am learning MDAnalysis through python-3.7. Would you please check my code and advise how to resolve the following error:
Traceback (most recent call last):
File "/home/pulokdeb/projects/def-sohrabz/pulokdeb/beluga_python/Closest_atom_Oxy_group.py", line 242, in <module>
eigen_value = iio.eigen_vals()
File "/home/pulokdeb/ENV/lib/python3.7/site-packages/MDAnalysis/core/topologyattrs.py", line 1347, in eigen_vals
com = atomgroup.center_of_mass(pbc=pbc)
NameError: name 'pbc' is not defined
The code (partial) is below:
def radius_of_gyration(group, pbc=False, **kwargs):
"""Radius of gyration.
Parameters
----------
pbc : bool, optional
If ``True``, move all atoms within the primary unit cell before
calculation. [``False``]
.. versionchanged:: 0.8 Added *pbc* keyword
"""
atomgroup = group.atoms
masses = atomgroup.masses
com = atomgroup.center_of_mass(pbc=pbc)
if pbc:
recenteredpos = atomgroup.pack_into_box(inplace=False) - com
else:
recenteredpos = atomgroup.positions - com
rog_sq = np.sum(masses * np.sum(recenteredpos**2,
axis=1)) / atomgroup.total_mass()
return np.sqrt(rog_sq)
transplants[GroupBase].append(
('radius_of_gyration', radius_of_gyration))
I changed a few lines (def_eif_vals) in topologyattrs.py file and got my results. Hope it works for my future simulations.
def shape_parameter(group, pbc=False, **kwargs):
"""Shape parameter.
See [Dima2004a]_ for background information.
Parameters
----------
pbc : bool, optional
If ``True``, move all atoms within the primary unit cell before
calculation. [``False``]
References
----------
.. [Dima2004a] Dima, R. I., & Thirumalai, D. (2004). Asymmetry
in the shapes of folded and denatured states of
proteins. *J Phys Chem B*, 108(21),
6564-6570. doi:`10.1021/jp037128y
<https://doi.org/10.1021/jp037128y>`_
.. versionadded:: 0.7.7
.. versionchanged:: 0.8 Added *pbc* keyword
"""
atomgroup = group.atoms
masses = atomgroup.masses
com = atomgroup.center_of_mass(pbc=pbc)
if pbc:
recenteredpos = atomgroup.pack_into_box(inplace=False) - com
else:
recenteredpos = atomgroup.positions - com
tensor = np.zeros((3, 3))
for x in range(recenteredpos.shape[0]):
tensor += masses[x] * np.outer(recenteredpos[x, :],
recenteredpos[x, :])
tensor /= atomgroup.total_mass()
eig_vals = np.linalg.eigvalsh(tensor)
shape = 27.0 * np.prod(eig_vals - np.mean(eig_vals)
) / np.power(np.sum(eig_vals), 3)
return shape
transplants[GroupBase].append(
('shape_parameter', shape_parameter))
def eigen_vals(group, pbc=False, **kwargs):
""" Changed by Pulok Deb
"""
atomgroup = group.atoms
masses = atomgroup.masses
com = atomgroup.center_of_mass(pbc=pbc)
if pbc:
recenteredpos = atomgroup.pack_into_box(inplace=False) - com
else:
recenteredpos = atomgroup.positions - com
tensor = np.zeros((3, 3))
for x in range(recenteredpos.shape[0]):
tensor += masses[x] * np.outer(recenteredpos[x, :],
recenteredpos[x, :])
tensor /= atomgroup.total_mass()
eig_vals = np.linalg.eigvalsh(tensor)
return eig_vals
transplants[GroupBase].append(
('eigen_vals', eigen_vals))
#warn_if_not_unique
#check_pbc_and_unwrap

Cryptic TypeError: 'decimal.Decimal' object cannot be interpreted as an integer

I am struggling to understand why this function apparently fails in the Jupyter Notebook, but not in the IPython shell:
def present_value( r, n, fv = None, pmt = None ):
'''
Function to compute the Present Value based on interest rate and
a given future value.
Arguments accepted
------------------
* r = interest rate,
which should be given in its original percentage, eg.
5% instead of 0.05
* n = number of periods for which the cash flow,
either as annuity or single flow from one present value
* fv = future value in dollars,
if problem is annuity based, leave this empty
* pmt = each annuity payment in dollars,
if problem is single cash flow based, leave this empty
'''
original_args = [r, n, fv, pmt]
dec_args = [Decimal( arg ) if arg != None
else arg
for arg in original_args
]
if dec_args[3] == None:
return dec_args[2] / ( ( 1 + ( dec_args[0] / 100 ) )**dec_args[1] )
elif dec_args[2] == None:
# annuity_length = range( 1, dec_args[1] + 1 )
# Not allowed to add a Decimal object
# with an integer and to use it
# in the range() function,
# so we dereference the integer from original_args
annuity_length = range( 1, original_args[1] + 1 )
# Apply discounting to each annuity payment made
# according to number of years left till end
all_compounded_pmt = [dec_args[3] * ( 1 / ( ( 1 + dec_args[0] / 100 ) ** time_left ) ) \
for time_left in annuity_length
]
return sum( all_compounded_pmt )
When I imported the module that this function resides in, named functions.py, using from functions import *, and then executed present_value(r=7, n=35, pmt = 11000), I got the error:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-93-c1cc587f7e27> in <module>()
----> 1 present_value(r=7, n=35, pmt = 11000)
/path_to_file/functions.py in present_value(r, n, fv, pmt)
73 if dec_args[3] == None:
74 return dec_args[2]/((1 + (dec_args[0]/100))**dec_args[1])
---> 75
76 elif dec_args[2] == None:
77 # annuity_length = range(1, dec_args[1]+1)
TypeError: 'decimal.Decimal' object cannot be interpreted as an integer
but in the IPython shell, evaluating this function it works perfectly fine:
In [42]: functions.present_value(r=7, n=35, pmt = 11000)
Out[42]: Decimal('142424.39530474029537')
Can anyone please help me with this really confusing and obscure issue?

scipy.optimize.fsolve 'proper array of floats' error

I need to compute the root of a function and I'm using scipy.optimize.fsolve. However when I call fsolve, sometimes it outputs an error that says 'Result from function call is not a proper array of floats.'
Here's an example of the inputs I'm using:
In [45]: guess = linspace(0.1,1.0,11)
In [46]: alpha_old = 0.5
In [47]: n_old = 0
In [48]: n_new = 1
In [49]: S0 = 0.9
In [50]: fsolve(alpha_eq,guess,args=(n_old,alpha_old,n_new,S0))
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
TypeError: array cannot be safely cast to required type
---------------------------------------------------------------------------
error Traceback (most recent call last)
/home/andres/Documents/UdeA/Proyecto/basis_analysis/<ipython-input-50-f1e9a42ba072> in <module>()
----> 1 fsolve(bb.alpha_eq,guess,args=(n_old,alpha_old,n_new,S0))
/usr/lib/python2.7/dist-packages/scipy/optimize/minpack.pyc in fsolve(func, x0, args, fprime, full_output, col_deriv, xtol, maxfev, band, epsfcn, factor, diag)
123 maxfev = 200*(n + 1)
124 retval = _minpack._hybrd(func, x0, args, full_output, xtol,
--> 125 maxfev, ml, mu, epsfcn, factor, diag)
126 else:
127 _check_func('fsolve', 'fprime', Dfun, x0, args, n, (n,n))
error: Result from function call is not a proper array of floats.
In [51]: guess = linspace(0.1,1.0,2)
In [52]: fsolve(alpha_eq,guess,args=(n_old,alpha_old,n_new,S0))
Out[52]: array([ 0.54382423, 1.29716005])
In [53]: guess = linspace(0.1,1.0,3)
In [54]: fsolve(alpha_eq,guess,args=(n_old,alpha_old,n_new,S0))
Out[54]: array([ 0.54382423, 0.54382423, 1.29716005])
There you can see that for 'guess' as defined in In[46] it outputs an error, however for 'guess' as defined in In[51] and in In[53] it works ok. As far as I know both In[46], In[51] and In[53] are the same type of arrays so what's the reason for the error I'm getting in In[50]?
Here are the functions I'm calling in case they're the reason of the problem:
def alpha_eq(alpha2,n1,alpha1,n2,S0):
return overlap(n1,alpha1,n2,alpha2) - S0
def overlap(n1,alpha1,n2,alpha2):
aux1 = sqrt((2.0*alpha1)**(2*n1+3)/factorial(2*n1+2))
aux2 = sqrt((2.0*alpha2)**(2*n2+3)/factorial(2*n2+2))
return aux1 * aux2 * factorial(n1+n2+2) / (alpha1+alpha2)**(n1+n2+3)
(the functions linspace, sqrt and factorial are imported from scipy)
This is a plot of the function for which I'm trying to find the roots.
plot
It seems to me like this is a bug of fsolve, however I want to make sure I'm not making a stupid mistake before reporting it.
If there's something wrong with my code please let me know. Thanks!
I have modified your overlap function for debugging as follows:
def overlap(n1,alpha1,n2,alpha2):
print n1, alpha1, n2, alpha2
aux1 = sqrt((2.0*alpha1)**(2*n1 + 3)/factorial(2*n1 + 2))
aux2 = sqrt((2.0*alpha2)**(2*n2 + 3)/factorial(2*n2 + 2))
ret = aux1 * aux2 * factorial(n1+n2+2) / (alpha1+alpha2)**(n1+n2+3)
print ret, ret.dtype
return ret
And when I try to reproduce your error, here's what happens:
>>> scipy.optimize.fsolve(alpha_eq,guess,args=(n_old,alpha_old,n_new,S0))
0 0.5 1 [ 0.1 0.19 0.28 0.37 0.46 0.55 0.64 0.73 0.82 0.91 1. ]
[ 0.11953652 0.34008953 0.54906314 0.71208678 0.82778065 0.90418052
0.95046505 0.97452352 0.98252708 0.97911263 0.96769965] float64
...
0 0.5 1 [ 0.45613162 0.41366639 0.44818267 0.49222515 0.52879856 0.54371741
0.50642005 0.28700652 -3.72580492 1.81152096 1.41975621]
[ 0.82368346+0.j 0.77371428+0.j 0.81503304+0.j
0.85916030+0.j 0.88922137+0.j 0.89992643+0.j
0.87149667+0.j 0.56353606+0.j 0.00000000+1.21228156j
0.75791881+0.j 0.86627491+0.j ] complex128
So in the process of solving your equation, the square root of a negative number is being calculated, which leads to the complex128 dtype and your error.
With your function, if you are only interested in the zeros, I think you can get rid of the sqrts if you raise S0 to the 4th power:
def alpha_eq(alpha2,n1,alpha1,n2,S0):
return overlap(n1,alpha1,n2,alpha2) - S0**4
def overlap(n1,alpha1,n2,alpha2):
aux1 = (2.0*alpha1)**(2*n1 + 3)/factorial(2*n1 + 2)
aux2 = (2.0*alpha2)**(2*n2 + 3)/factorial(2*n2 + 2)
ret = aux1 * aux2 * factorial(n1+n2+2) / (alpha1+alpha2)**(n1+n2+3)
return ret
And now:
>>> scipy.optimize.fsolve(alpha_eq,guess,args=(n_old,alpha_old,n_new,S0))
array([ 0.92452239, 0.92452239, 0.92452239, 0.92452239, 0.92452239,
0.92452239, 0.92452239, 0.92452239, 0.92452239, 0.92452239,
0.92452239])