Which of the following library is used to run a Student's t-test using the scipy library in Python.
?
a = [15, 12, 7, 98]
b = [2, 20, 8, 28]
stat, p = ttest_ind(a, b)
print(stat,p)
Options:
from scipy.ttest_ind import ttest_ind
from student.ttest_ind import ttest_ind
from scipy.statistics import ttest_ind
from scipy.ttest_ind import ttest_ind
Related
I am working to adapt a python2.7 code that uses keras and tensorflow to implement a CNN but looks like the keras API has changed a little bit since when the original code was idealized. I keep getting an error about "Negative dimension after subtraction" and I can not find out what is causing it.
Unfortunately I am not able to provide an executable piece of code because I was not capable of make the original code works, but the repository containing all the source files can be found here.
The piece of code:
from keras.callbacks import EarlyStopping
from keras.layers.containers import Sequential
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras.layers.core import Reshape, Flatten, Dropout, Dense
from keras.layers.embeddings import Embedding
from keras.models import Graph
from keras.preprocessing import sequence
filter_lengths = [3, 4, 5]
self.model = Graph()
'''Embedding Layer'''
self.model.add_input(name='input', input_shape=(max_len,), dtype=int)
self.model.add_node(Embedding(
max_features, emb_dim, input_length=max_len), name='sentence_embeddings', input='input')
'''Convolution Layer & Max Pooling Layer'''
for i in filter_lengths:
model_internal = Sequential()
model_internal.add(
Reshape(dims=(1, self.max_len, emb_dim), input_shape=(self.max_len, emb_dim))
)
model_internal.add(Convolution2D(
nb_filters, i, emb_dim, activation="relu"))
model_internal.add(
MaxPooling2D(pool_size=(self.max_len - i + 1, 1))
)
model_internal.add(Flatten())
self.model.add_node(model_internal, name='unit_' + str(i), input='sentence_embeddings')
What I have tried:
m = tf.keras.Sequential()
m.add(tf.keras.Input(shape=(max_len, ), name="input"))
m.add(tf.keras.layers.Embedding(max_features, emb_dim, input_length=max_len))
filter_lengths = [ 3, 4, 5 ]
for i in filter_lengths:
model_internal = tf.keras.Sequential(name=f'unit_{i}')
model_internal.add(
tf.keras.layers.Reshape(( 1, max_len, emb_dim ), input_shape=( max_len, emb_dim ))
)
model_internal.add(
tf.keras.layers.Convolution2D(100, i, emb_dim, activation="relu")
)
model_internal.add(
tf.keras.layers.MaxPooling2D(pool_size=( max_len - i + 1, 1 ))
)
model_internal.add(
tf.keras.layers.Flatten()
)
m.add(model_internal)
I do not expect a complete solution, what I am really trying to understand is what is the cause to the following error:
Negative dimension size caused by subtracting 3 from 1 for '{{node conv2d_5/Conv2D}} = Conv2D[T=DT_FLOAT, data_format="NHWC", dilations=[1, 1, 1, 1], explicit_paddings=[], padding="VALID", strides=[1, 200, 200, 1], use_cudnn_on_gpu=true](Placeholder, conv2d_5/Conv2D/ReadVariableOp)' with input shapes: [?,1,300,200], [3,3,200,100].
I want to test the explainability of a multiclass semantic segmentation model, deeplab_v3plus with shap to know which features contribute the most to semantic classification. However I have a ValueError: max_evals=500 is too low when running my file, and I struggle to understand the reason.
import glob
from PIL import Image
import torch
from torchvision import transforms
from torchvision.utils import make_grid
import torchvision.transforms.functional as tf
from deeplab import deeplab_v3plus
import shap
def test(args):
# make a video prez
model = deeplab_v3plus('resnet101', num_classes=args.nclass, output_stride=16, pretrained_backbone=True)
model.load_state_dict(torch.load(args.seg_file,map_location=torch.device('cpu'))) # because no gpu available on sandbox environnement
model = model.to(args.device)
model.eval()
explainer = shap.Explainer(model)
with torch.no_grad():
for i, file in enumerate(args.img_folder):
img = img2tensor(file, args)
pred = model(img)
print(explainer(img))
if __name__ == '__main__':
class Arguments:
def __init__(self):
self.device = torch.device("cuda:1" if torch.cuda.is_available() else "cpu")
self.seg_file = "Model_Woodscape.pth"
self.img_folder = glob.glob("test_img/*.png")
self.mean = [0.485, 0.456, 0.406]
self.std = [0.229, 0.224, 0.225]
self.h, self.w = 483, 640
self.nclass = 10
self.cmap = {
1: [128, 64, 128], # "road",
2: [69, 76, 11], # "lanemarks",
3: [0, 255, 0], # "curb",
4: [220, 20, 60], # "person",
5: [255, 0, 0], # "rider",
6: [0, 0, 142], # "vehicles",
7: [119, 11, 32], # "bicycle",
8: [0, 0, 230], # "motorcycle",
9: [220, 220, 0], # "traffic_sign",
0: [0, 0, 0] # "void"
}
args = Arguments()
test(args)
But it returns:
(dee_env) jovyan#jupyter:~/use-cases/Scene_understanding/Code_Woodscape/deeplab_v3+$ python test_shap.py
BILINEAR is deprecated and will be removed in Pillow 10 (2023-07-01). Use Resampling.BILINEAR instead.
Traceback (most recent call last):
File "/home/jovyan/use-cases/Scene_understanding/Code_Woodscape/deeplab_v3+/test_shap.py", line 85, in <module>
test(args)
File "/home/jovyan/use-cases/Scene_understanding/Code_Woodscape/deeplab_v3+/test_shap.py", line 37, in test
print(explainer(img))
File "/home/jovyan/use-cases/Scene_understanding/Code_Woodscape/deeplab_v3+/dee_env/lib/python3.9/site-packages/shap/explainers/_permutation.py", line 82, in __call__
return super().__call__(
File "/home/jovyan/use-cases/Scene_understanding/Code_Woodscape/deeplab_v3+/dee_env/lib/python3.9/site-packages/shap/explainers/_explainer.py", line 266, in __call__
row_result = self.explain_row(
File "/home/jovyan/use-cases/Scene_understanding/Code_Woodscape/deeplab_v3+/dee_env/lib/python3.9/site-packages/shap/explainers/_permutation.py", line 164, in explain_row
raise ValueError(f"max_evals={max_evals} is too low for the Permutation explainer, it must be at least 2 * num_features + 1 = {2 * len(inds) + 1}!")
ValueError: max_evals=500 is too low for the Permutation explainer, it must be at least 2 * num_features + 1 = 1854721!
In the source code it looks like it's because I don't give enough arguments. I only have three images in my test_img/* folder, is that why?
I have the same issue. A possible solution I found which seems to be working for my case is to replace this line
explainer = shap.Explainer(model)
With this line
explainer = shap.explainers.Permutation(model, max_evals = 1854721)
shap.Explainer by default has algorithm='auto'. From the documentation: shape.Explainer
By default the “auto” options attempts to make the best choice given
the passed model and masker, but this choice can always be overriden
by passing the name of a specific algorithm.
Since 'permutation' has been selected you can directly use shap.explainers.Permutation and set max_evals to the value suggested in the error message above.
Given the high number of your use case, this might take a really long time. I would suggest to use an easier model just for testing the above solution.
I am trying to write some data that I extracted from an excel file to a '.mat' file. So far, I have converted the extracted data into an array and converted this array to a dictionary before writing to a .mat file. While the conversions to the array and dictionary seem fine, when I create and write to a .mat file, the data seems corrupted. Here is my code:
import pandas as pd
file_location = '/Users/manmohidake/GoogleDrive/Post_doc/Trial_analysis/1_IndoorOutdoor.xlsx'
mydata = pd.read_excel(file_location,na_values = "Missing", sheet_name='Sheet1', skiprows = 1, usecols="F,K,Q")
import numpy
#Convert data to array
array = mydata.to_numpy()
import scipy.io
import os
destination_folder_path = '/Users/manmohidake/Google Drive/Post_doc/Trial_analysis/'
scipy.io.savemat(os.path.join(destination_folder_path,'trial1.mat'), {'array':array})
I don't really know what's gone wrong. When I open the .mat file, it. looks like this
Matlab file
In [1]: from scipy import io
In [2]: arr = np.arange(12).reshape(4,3)
In [3]: io.savemat('test.mat',{'array':arr})
In [4]: io.loadmat('test.mat')
Out[4]:
{'__header__': b'MATLAB 5.0 MAT-file Platform: posix, Created on: Mon Sep 20 11:36:48 2021',
'__version__': '1.0',
'__globals__': [],
'array': array([[ 0, 1, 2],
[ 3, 4, 5],
[ 6, 7, 8],
[ 9, 10, 11]])}
In Octave
>> cd mypy
>> load test.mat
>> array
array =
0 1 2
3 4 5
6 7 8
9 10 11
I am using Pytesseract in python, to recognize only numbers of 1, 2 and 3 digits, (any) in a screenshot of the screen, but sometimes it is conjugated to numbers with letters, (a = 4) which is a problem. I think the numbers are very smallis the number in screenshot
import pytesseract
from PIL import Image
import cv2
import pyautogui
pytesseract.pytesseract.tesseract_cmd =r"C:\Users\Lenovo\AppData\Local\Tesseract-
OCR\tesseract.exe"
while True:
answer2 = pyautogui.screenshot("answer2.png",region=(456, 51, 28, 14))
im = cv2.imread("answer2.png",1)
answer2 = pytesseract.image_to_string(Image.fromarray(im), config="--psm 6")
answer2=float(answer2)
print(answer2)
I just started programming and recently I'm working in ipython notebook with networkx. This code below is working perfectly if you run it, but if you uncomment #G.add_edge(2, 8, egdes=6) it gives you the error NetworkXError: Node 8 has no position. Why does it only work until the sixth node?
import networkx as nx
import matplotlib.pyplot as plt
import pylab
%matplotlib inline
pos=nx.spring_layout(G)
G = nx.Graph()
G.add_edge(1, 2, egdes=1)
G.add_edge(1, 3, egdes=2)
G.add_edge(1, 4, egdes=3)
G.add_edge(1, 5, egdes=4)
G.add_edge(1, 6, egdes=5)
#G.add_edge(2, 8, egdes=6)
nx.draw(G,pos)
edge_labels=dict([((fe,se,),e['egdes'])
for fe,se,e in G.edges(data=True)])
nx.draw_networkx_edge_labels(G,pos,edge_labels)
pylab.show()
I hope one of you guys can help me, thanks in advance!
You need to create the node positions
pos=nx.spring_layout(G)
after you have built the graph (added all edges and nodes) and before you draw it.