Traffic Sign detection and Recognition using Neural networks - neural-network

Question

I wanted to detect and recognize traffic signs from a video feed. I used the Tensorflow ML framework for recognition of signs and used haar classifier for detection of signs.
Here is the code:
import cv2
import numpy as np
import tensorflow as tf
import os,time
import threading
# constants
IMAGE_SIZE = 200.0
MATCH_THRESHOLD = 3
def SignRecognizer():
#to neglect all tensorflow compilation warnings
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
#path to the blob
image_path='/root/Desktop/blob.jpg'
#read the image data
image_data = tf.gfile.FastGFile(image_path,'rb').read()
#load label file,strip off carriage return \n
label_lines= [line.rstrip() for line in tf.gfile.GFile("/root/Desktop/another_model/retrained_labels.txt")]
#unpersists graph from file
with tf.gfile.FastGFile("/root/Desktop/another_model/retrained_graph.pb",'rb') as f:
graph_def=tf.GraphDef()
graph_def.ParseFromString(f.read())
_=tf.import_graph_def(graph_def,name='')
with tf.Session() as sess:
#feed the image_data as input to the graph and get the first prediction
softmax_tensor=sess.graph.get_tensor_by_name("final_result:0")
predictions = sess.run(softmax_tensor,\
{'DecodeJpeg/contents:0':image_data})
#sort to show labels of first prediction in order of confidence
top_k=predictions[0].argsort()[-len(predictions[0]):][::-1]
for node_id in top_k:
human_string=label_lines[node_id]
print("%s"%(human_string))
break
roundabout_cascade = cv2.CascadeClassifier("/root/Desktop/tsp/haarcascade_roundabout.xml")
videocapture = cv2.VideoCapture(0)
scale_factor=1.3
while 1:
ret,pic = videocapture.read()
# do roundabout detection on street image
gray = cv2.cvtColor(pic,cv2.COLOR_RGB2GRAY)
signs = roundabout_cascade.detectMultiScale(pic,scaleFactor=1.4,minNeighbors=6)
# initialize ORB and BFMatcher
orb = cv2.ORB_create()
bf = cv2.BFMatcher(cv2.NORM_HAMMING,crossCheck=True)
# find the keypoints and descriptors for roadsign image
roadsign = cv2.imread("/root/Desktop/tsp/roundabout.jpg",0)
kp_r,des_r = orb.detectAndCompute(roadsign,None)
for (x,y,w,h) in signs:
#cv2.rectangle(pic,(x,y),(x+w,y+h),(255,0,0),2)
# obtain object from street image
obj = gray[y:y+h,x:x+w]
color_image=pic[y:y+h,x:x+w]
cv2.imwrite("/root/Desktop/blob.jpg",color_image)
cv2.imshow('blob', color_image)
#start a new thread and run SignRecognizer on it
t=threading.Thread(name="SignRecognizer",target=SignRecognizer)
#set the thread as a daemon to prevent blocking of the main program
t.setDaemon(True)
t.start()
ratio = IMAGE_SIZE / obj.shape[1]
obj = cv2.resize(obj,(int(IMAGE_SIZE),int(obj.shape[0]*ratio)))
# find the keypoints and descriptors for object
kp_o, des_o = orb.detectAndCompute(obj,None)
if len(kp_o) == 0 or des_o == None:
continue
# match descriptors
matches = bf.match(des_r,des_o)
# draw object on street image, if threshold met
if(len(matches) >= MATCH_THRESHOLD):
cv2.rectangle(pic,(x,y),(x+w,y+h),(255,0,0),2)
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(pic,'Roundabout sign',(x,y),font,1,(255,255,255),1,cv2.LINE_AA)
cv2.imshow('roundabout_signs',pic)
k = cv2.waitKey(30) & 0xFF
if k==2:
break
cv2.waitKey(0)
cv2.destroyAllWindows()
The SignRecognizer function reads the blob image file and recognizes the sign using the model I created using tensorflow ML Framework.
I used VideoCapture(0) to start the webcam and simulate a live video feed.
I also used OpenCV's ORB ( Oriented FAST and rotated BRIEF) to remove false positives.
I used threading module to run the SignRecognizer on another thread and set it as a daemon so that the main pgm. wasn't blocked during recognition.
Everything works great but there seems to be a little lag inspite of using threading module.Is there any way to make it lag free?