I am begin in Keras, and I need to implement some type:
A CNN with 1 channel input and 2 channel as output being both regressions?
My solution
dataset = "./dataset/img/"
dataset_file = "./dataset/gt.txt"
def model_network(dense_y, dense_x):
main_input = Input(shape=shape_img[1:], name="main_input")
flow = Convolution2D(32, 3, 3, border_mode='same')(main_input)
flow = Activation('relu')(flow)
flow = Convolution2D(32, 3, 3)(flow)
flow = Activation('relu')(flow)
flow = MaxPooling2D(pool_size=(2, 2))(flow)
flow = Dropout(0.25)(flow)
flow = Convolution2D(64, 3, 3, border_mode='same')(flow)
flow = Activation('relu')(flow)
flow = Convolution2D(64, 3, 3)(flow)
flow = Activation('relu')(flow)
flow = MaxPooling2D(pool_size=(2, 2))(flow)
flow = Dropout(0.25)(flow)
flow = Convolution2D(512, 3, 3, border_mode='same')(flow)
flow = Activation('relu')(flow)
flow = Convolution2D(512, 3, 3)(flow)
flow = Activation('relu')(flow)
flow = MaxPooling2D(pool_size=(2, 2))(flow)
flow = Dropout(0.25)(flow)
flow = Flatten()(flow)
cod_x = Dense(100)(flow)
output_x = Dense(dense_x, activation='sigmoid', name="output_x")(cod_x)
cod_y = Dense(100)(flow)
output_y = Dense(dense_y, activation='sigmoid', name="output_y")(cod_y)
model = Model(input=[main_input], output=[output_x, output_y])
return model
def train_model(model, path):
# let's train the model using SGD + momentum (how original).
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
print("... compiling")
model.compile(optimizer=sgd,
loss_weights=[0.5, 0.5],
loss='categorical_crossentropy')
print("... training")
model.fit_generator(generate_array_files(path), samples_per_epoch=500, nb_val_samples=60, nb_epoch=10)
save_model(model)
if __name__ == '__main__':
path = "dataset_names.csv"
model_tracking = model_network(length_y, length_x)
train_model(model_tracking, path=path)
Related
dosen't work when i try to excuted it's stock not work
def calculate_utility_cost(visitors) :
movies = ['Gifted Hands', 'Legends of the Fall', 'Patch Adams',
'The Sixth Sense', 'A Beautiful Mind']
visitors_per_movie = np.repeat(0, len(movies))
pred_bill_per_week = 100000/4
pred_visitors_per_week = 14413
return pred_bill_per_week / pred_visitors_per_week * visitors
utility_total_cost1 = int(input(calculate_utility_cost))
utility_total_cost2 = (visitors_per_day)
utility_total_cost= sum = utility_total_cost1 +
utility_total_cost2
print(utility_total_cost)
The problem is dataloader is returning the wrong class for correspond image?
for example if I print the class_to_idx from the train_loader, when batch size is 1I was expecting to get one class per batch, but currently it’s returning all the classes which is 15 classes per image.
In this case, the classes are folder class (all images exist in one folder belongs to one class)
snippet is here:(this is a function to return the class from the folder name dir)
import os
def find_classes(dir): # Finds the class folders in a dataset, dir (string): Root directory path.
classes = [d.name for d in os.scandir(dir) if d.is_dir()]
classes.sort()
class_to_idx = {classes[i]: i for i in range(len(classes))}
return classes, class_to_idx
here is the main snippet for create a custom dataset and dataloder
def main():
class CustomDataset(Dataset):
def __init__(self, image_paths, classes, class_to_id):
self.image_paths = image_paths
self.transforms = transforms.ToTensor()
classes, class_to_id = find_classes('D:/Neda/Echo_View_Classification/avi_images/')
self.classes = classes
self.class_to_idx = class_to_idx
def __getitem__(self, index):
image = Image.open(self.image_paths[index])
t_image = image.convert('L')
t_image = self.transforms(t_image)
class_to_idx = self.class_to_idx
return t_image, class_to_idx, self.image_paths[index]
def __len__(self):
return len(self.image_paths)
folder_data = glob.glob("D:\\Neda\\Echo_View_Classification\\avi_images\\*\\*.png") # no augmnetation
#numpy.savetxt('distribution_class.csv', numpy.c_[folder_data], fmt=['%s'], comments='', delimiter = ",")
#split these path using a certain percentage
len_data = len(folder_data)
print("count of dataset: ", len_data)
split_1 = int(0.6 * len(folder_data))
split_2 = int(0.8 * len(folder_data))
folder_data.sort()
train_image_paths = folder_data[:split_1]
print("count of train images is: ", len(train_image_paths))
numpy.savetxt('im_training_path_1.csv', numpy.c_[train_image_paths], fmt=['%s'], comments='', delimiter = ",")
valid_image_paths = folder_data[split_1:split_2]
print("count of validation image is: ", len(valid_image_paths))
numpy.savetxt('im_valid_path_1.csv', numpy.c_[valid_image_paths], fmt=['%s'], comments='', delimiter = ",")
test_image_paths = folder_data[split_2:]
print("count of test images is: ", len(test_image_paths))
numpy.savetxt('im_testing_path_1.csv', numpy.c_[test_image_paths], fmt=['%s'], comments='', delimiter = ",")
classes = ['1_PLAX_1_PLAX_full',
'1_PLAX_2_PLAX_valves',
'1_PLAX_4_PLAX_TV',
'2_PSAX_1_PSAX_AV',
'2_PSAX_2_PSAX_LV',
'3_Apical_1_MV_LA_IAS',
'3_Apical_2_A2CH',
'3_Apical_3_A3CH',
'3_Apical_5_A5CH',
'4_A4CH_1_A4CH_LV',
'4_A4CH_2_A4CH_RV',
'4_Subcostal_1_Subcostal_heart',
'4_Subcostal_2_Subcostal_IVC',
'root_5_Suprasternal',
'root_6_OTHER']
class_to_idx = {'1_PLAX_1_PLAX_full': 0,
'1_PLAX_2_PLAX_valves': 1,
'1_PLAX_4_PLAX_TV': 2,
'2_PSAX_1_PSAX_AV': 3,
'2_PSAX_2_PSAX_LV': 4,
'3_Apical_1_MV_LA_IAS': 5,
'3_Apical_2_A2CH': 6,
'3_Apical_3_A3CH': 7,
'3_Apical_5_A5CH': 8,
'4_A4CH_1_A4CH_LV': 9,
'4_A4CH_2_A4CH_RV': 10,
'4_Subcostal_1_Subcostal_heart': 11,
'4_Subcostal_2_Subcostal_IVC': 12,
'root_5_Suprasternal': 13,
'root_6_OTHER': 14}
train_dataset = CustomDataset(train_image_paths, class_to_idx, classes)
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=1, shuffle=False, num_workers=0)
valid_dataset = CustomDataset(valid_image_paths, class_to_idx, classes)
valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=1, shuffle=False, num_workers=0)
test_dataset = CustomDataset(test_image_paths, class_to_idx, classes)
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=0)
dataLoaders = {
'train': train_loader,
'valid': valid_loader,
'test': test_loader,
}
I think ImageFolder from torchvision.datasets will help you in loading your data.
I can not find a library to use PAM (K-medoids) in Pyspark.
I have found this in Scala :
https://gist.github.com/erikerlandson/c3c35f0b1aae737fc884
And this issue in Spark which was resolved in 2016 :
https://issues.apache.org/jira/browse/SPARK-4510
https://github.com/apache/spark/pull/3382
But it seems not to be working and this is not included in the mllib documentation :
http://spark.apache.org/docs/2.0.0/api/python/pyspark.mllib.html#module-pyspark.mllib.clustering
Does anyone knows any library for PAM in Pyspark ?
Thank you
I actually had a go at this the other day for fun. Can't say much about performance as I'm quite new to spark. But here is KMedoids with K++ seeding:
# (c) 2020 Jonathan Kelsey
# This code is licensed under MIT license
from pyspark.sql import functions as F
import pyspark
import numpy as np
import sys
def seed_kernel(data_broadcast, data_id_value, centeroids, k, metric):
data = data_broadcast.value
point = data_id_value[1]
min_distance = sys.maxsize
for j in range(len(centeroids)):
distance = metric(point, data[centeroids[j]])
min_distance = min(min_distance, distance)
return min_distance
def seed_clusters(data_broadcast, data_frame, k, metric):
data = data_broadcast.value
centeroids = list(np.random.choice(data.shape[0], 1, replace=False))
for i in range(k - 1):
print("clusterSeed", i)
distances = []
mK = data_frame.rdd.map(lambda data_id_value: seed_kernel(data_broadcast, data_id_value, centeroids, k, metric))
mK_collect = mK.collect()
distances = np.array(mK_collect)
next_centeroid = np.argmax(distances)
centeroids.append(next_centeroid)
print("centeroids", centeroids)
return centeroids
def nearest_centeroid_kernel(data_id_value, centeroid_id_values, metric):
_, data_value = data_id_value
data_np = np.asarray(data_value)
distances = []
for _, centeroid_value in centeroid_id_values:
centeroid_np = np.asarray(centeroid_value)
distance = metric(data_np, centeroid_np)
distances.append(distance)
distances = np.asarray(distances)
closest_centeroid = np.argmin(distances)
return int(closest_centeroid)
def optimise_cluster_membership_spark(data, data_frame, n, metric, intital_cluster_indices=None):
data_shape = data.shape
data_rdd = data_frame.rdd
data_length = data_shape[0]
if intital_cluster_indices is None:
index = np.random.choice(data_length, n, replace=False)
else:
index = intital_cluster_indices
list_index = [int(i) for i in list(index)]
centeroid_id_values = [(i,data[index[i]]) for i in range(len(index))]
data_rdd = data_rdd.filter(lambda data_id_value: int(data_id_value["id"]) not in list_index)
associated_cluster_points = data_rdd.map(lambda data_id_value: (data_id_value[0],nearest_centeroid_kernel(data_id_value, centeroid_id_values, metric)))
clusters = associated_cluster_points.toDF(["id", "bestC"]).groupBy("bestC").agg(F.collect_list("id").alias("cluster"))
return index, clusters
def cost_kernel(data_broadcast, test_centeroid, cluster_data, metric):
data = data_broadcast.value
cluster = np.asarray(cluster_data)
cluster_length = cluster.shape[0]
feature_length = data.shape[1]
test_centeroid_column = np.zeros(shape=(cluster_length, feature_length), dtype=data.dtype)
new_cluster_column = np.zeros(shape=(cluster_length, feature_length), dtype=data.dtype)
for i in range(0, cluster_length):
new_cluster_column[i] = data[cluster[i]]
test_centeroid_column[i] = data[int(test_centeroid)]
pairwise_distance = metric(new_cluster_column, test_centeroid_column)# (np.absolute(new_cluster_column-test_centeroid_column).sum(axis=1))# metric(new_cluster_column, test_centeroid_column)
cost = np.sum(pairwise_distance)
return float(cost) #new_cluster_column.shape[1]
def optimise_centroid_selection_spark(data_broadcast, data_frame, centeroids, clusters_frames, metric):
data = data_broadcast.value
new_centeroid_ids = []
total_cost = 0
for cluster_idx in range(len(centeroids)):
old_centeroid = centeroids[cluster_idx]
cluster_frame = clusters_frames.filter(clusters_frames.bestC == cluster_idx).select(F.explode(clusters_frames.cluster))
cluster_data = cluster_frame.collect()
if cluster_data:
cluster_data = [cluster_data[i].col for i in range(len(cluster_data))]
else:
cluster_data = []
cost_data = cluster_frame.rdd.map(lambda point_id: (point_id[0], cost_kernel(data_broadcast, point_id[0], cluster_data, metric)))
cost = cost_data.map(lambda point_id_cost: point_id_cost[1]).sum()
total_cost = total_cost + cost
point_result = cost_data.sortBy(lambda point_id_cost: point_id_cost[1]).take(1)
if (point_result):
best_point = point_result[0][0]
else:
best_point = old_centeroid
new_centeroid_ids.append(best_point)
return (new_centeroid_ids, total_cost)
def validate_metric(metric):
if (metric == "euclidean" or metric == "hamming"):
return True
if isinstance(metric, dict) == False:
return "Metric is not a dictionary. And not a known string 'euclidean' or 'hamming'"
metric_keys = metric.keys()
if "point" not in metric_keys or "vector" not in metric_keys:
return "Metric does not contain a member function for 'point' and/or 'point'."
if callable(metric["point"]) == False or callable(metric["vector"]) == False:
return "Metric.point and/or Metric.vector are not callable functions."
if (metric["point"].__code__.co_argcount != 2 and metric["vector"].__code__.co_argcount != 2):
return "Metric.point and/or Metric.vector do not both have 2 arguments."
return True
# pre-defined metrics
#vector metrics
def hamming_vector(stack1, stack2):
return (stack1 != stack2).sum(axis=1)
def euclidean_vector(stack1, stack2):
#return (np.absolute(stack2-stack1)).sum(axis=1)
return np.sqrt(((stack2-stack1)**2).sum(axis=1))
# point metrics
def hamming_point(p1, p2):
return np.sum((p1 != p2))
def euclidean_point(p1, p2):
return np.sqrt(np.sum((p1 - p2)**2))
def fit(sc, data, n_clusters = 2, metric = "euclidean", seeding = "heuristic"):
metric_valid = validate_metric(metric)
if metric_valid == True:
if metric == "euclidean":
point_metric = euclidean_point
vector_metric = euclidean_vector
elif metric == "hamming":
point_metric = hamming_point
vector_metric = hamming_vector
else:
point_metric = metric["point"]
vector_metric = metric["vector"]
else:
print(metric_valid)
return
data_np = np.asarray(data)
data_broadcast = sc.broadcast(data_np)
seeds = None
data_frame = sc.parallelize(data).zipWithIndex().map(lambda xy: (xy[1],xy[0])).toDF(["id", "vector"]).cache()
if (seeding == "heuristic"):
seeds = list(seed_clusters(data_broadcast, data_frame, n_clusters, point_metric))
last_centeroids, last_clusters = optimise_cluster_membership_spark(data_np, data_frame, n_clusters, point_metric, seeds)
last_cost = float('inf')
iteration = 0
escape = False
while not escape:
iteration = iteration + 1
current_centeroids, current_cost = optimise_centroid_selection_spark(data_broadcast, data_frame, last_centeroids, last_clusters, vector_metric)
current_centeroids, current_clusters = optimise_cluster_membership_spark(data_np, data_frame, n_clusters, point_metric, current_centeroids)
print((current_cost<last_cost, current_cost, last_cost, current_cost - last_cost))
if (current_cost<last_cost):
print(("iteration",iteration,"cost improving...", current_cost, last_cost, current_centeroids))
last_cost = current_cost
last_centeroids = current_centeroids
last_clusters = current_clusters
else:
print(("iteration",iteration,"cost got worse or did not improve", current_cost, last_cost))
escape = True
bc = last_clusters.sort("bestC", ascending=True).collect()
unpacked_clusters = [bc[i].cluster for i in range(len(bc))]
return (last_centeroids, unpacked_clusters)
I used some sample data from pyclustering as a sanity check:
from pyclustering.cluster import cluster_visualizer
from pyclustering.utils import read_sample
from pyclustering.samples.definitions import FCPS_SAMPLES
from pyclustering.samples.definitions import SIMPLE_SAMPLES
sample = read_sample(FCPS_SAMPLES.SAMPLE_GOLF_BALL)
bestCentroids, bestClusters = fit(sc, sample, 9)
visualizer = cluster_visualizer()
visualizer.append_clusters(bestClusters, sample)
visualizer.show()
Your best choice is to adapt this Python implementation into Scala so you take advance of RDD partitions and distributed computation.
https://github.com/letiantian/kmedoids/blob/master/kmedoids.py
i would like to visualize what i will get after concatenation of map and struct in matlab , for instance let us consider following Map Container
ticketMap = containers.Map(...
{'2R175', 'B7398', 'A479GY', 'NZ1452'}, ...
{'James Enright', 'Carl Haynes', 'Sarah Latham', ...
'Bradley Reid'});
key/value structure of this map is clear for me, now let us suppose we have following structure
s1.ticketNum = '2S185'; s1.destination = 'Barbados';
s1.reserved = '06-May-2008'; s1.origin = 'La Guardia';
s2.ticketNum = '947F4'; s2.destination = 'St. John';
s2.reserved = '14-Apr-2008'; s2.origin = 'Oakland';
s3.ticketNum = 'A479GY'; s3.destination = 'St. Lucia';
s3.reserved = '28-Mar-2008'; s3.origin = 'JFK';
s4.ticketNum = 'B7398'; s4.destination = 'Granada';
s4.reserved = '30-Apr-2008'; s4.origin = 'JFK';
s5.ticketNum = 'NZ1452'; s5.destination = 'Aruba';
s5.reserved = '01-May-2008'; s5.origin = 'Denver';
we have 5 structure with different fields, now following commands
seatingMap = containers.Map( ...
{'23F', '15C', '15B', '09C', '12D'}, ...
{s5, s1, s3, s4, s2});
make sense for me because for instance using key 23F i can access fields of s1 structure, for instance
>> seatingMap('23F').origin
ans =
'Denver'
all those parts are clear for me, now Using ticketMap and seatingMap together, you can find the name of the person who has reserved seat 15B
ticket = seatingMap('15B').ticketNum;
passenger = ticketMap(ticket)
but is that optimal way?thanks in advance
I'm trying to code a GUI with two login functions and then a third window with a bunch of widgets. The issue I've run into is since I want to take entry values and retrieve them from another function, I figured I should use a Class. However, when I'm running my code, it seems to pop open all of the other Functions in the starting class.
import sys
from tkinter import *
class Gui():
def __init__(self, root):
self.root=root
self.entry = Entry(root)
stvar=StringVar()
stvar.set("one")
self.canvas=Canvas(root, width=300, height=300, background='white')
self.canvas.grid(row=1,column=0, columnspan = 4)
frame = Frame(self.root)
frame.grid(row=2,column=0, sticky="s")
frame2 = Frame(self.root)
frame2.grid(row=0,column=0, sticky = "n")
self.option=OptionMenu(frame, stvar, "one", "two", "three")
label1=Label(frame, text="Stock Mean:").grid(row=2,column=0, sticky="nw")
label2=Label(frame2, text="Stocks").grid(row=0,column=0,sticky = "w")
self.option=OptionMenu(frame2, stvar, "StockOne", "StockTwo", "StockThree").grid(row=1, column=0, sticky = "w")
label3= Label(frame, text="Std Variance").grid(row=2, column=1)
label4= Label(frame, text="Buy Price").grid(row=2, column=2)
label5=Label(frame, text="Sell Price").grid(row=2,column=3)
label6= Label(frame2, text="Auto/Manual").grid(row=0, column=3,sticky= "e")
labelSpace1 = Label(frame2, text= " ").grid(row=0, column = 1, columnspan = 2)
label7 = Label(frame, text = "Sample Mean").grid(row = 3, column = 0)
label8 = Label(frame, text = "Sample Std Variance").grid(row = 3, column = 1)
label9 = Label(frame, text = "Sample Buy Price").grid(row = 3, column = 2)
label10 = Label(frame, text = "Sample Sell Price").grid(row = 3, column = 3)
class Verification():
def __init__(self):
##First Window
self.master = master
label1 = Label(self.master, text = "Username:")
label1.pack()
user = Entry(self.master)
user.pack()
label2 = Label(self.master, text = "Password:")
label2.pack()
password = Entry(self.master)
password.pack()
button = Button(self.master, text = "Login", command = Verification.verify1(self))
self.master.title("Stock Program")
self.master.geometry("400x500")
button.pack()
##self.master.mainloop()
def verify1(self):
self.root1=Toplevel(self.master)
self.root1.title("Stock Broker Login")
self.root1.geometry("500x500")
##Broker Menu
variable = StringVar(self.root1)
variable.set("TestOne")
OPTIONS = ["One" , "Two", "Three", "Four"]
self.m = OptionMenu(self.root1, variable, OPTIONS)
self.m.pack()
##Login for Broker Account
label3 = Label(self.root1, text = "Username:")
label3.pack()
self.user2 = Entry(self.root1)
self.user2.pack()
label4 = Label(self.root1, text = "Password:")
label4.pack()
self.password2 = Entry(self.root1)
self.password2.pack()
self.user2info = self.user2.get()
self.pass2info = self.password2.get()
button2 = Button(self.root1, text = "Login", command =Verification.verify2(self) )
button2.pack()
## button3 = Button(self.root1, text = "Close", command = Verification.closewindow(self))
## button3.pack()
## def closewindow(self):
## self.master.destroy()
def verify2(self):
##if (self.user2info)=="Name":
self.GraphWindow()
##print (self.pass2info)
def GraphWindow(self):
self.root2 =Tk()
gui =Gui(self.root2)
##self.root2.mainloop()
if __name__ == '__main__':
master=Tk()
start = Verification()
master.mainloop()
The first code I used (Which avoided using this Class) did not run into this issue, so the secondary login window would only pop open if the first Login button was trigger( Which would trigger the command Verify1).
Does anyone know how I can prevent the other functions from triggering? Thanks!
In your __init__ method you are calling the methods that you're attempting to use as callbacks, rather than simply passing the function object. EX:
command = Verification.verify1(self)
Should instead be
command = self.verify1
You make the same mistake in verify1 with your binding to verify2