I want to use object detection using tensorflow lite in order to detect a clear face or a covered face where the statement "door opens" is printed when a clear face is detected. I could run this code smoothly previously but later after rebooting raspberry pi 4, although the tensorflow lite runtime is initialized, the raspberry pi 4 disconnects with the ssh completely. The following is the code:
######## Webcam Object Detection Using Tensorflow-trained Classifier #########
#
# Author: Evan Juras
# Date: 10/27/19
# Description:
# This program uses a TensorFlow Lite model to perform object detection on a live webcam
# feed. It draws boxes and scores around the objects of interest in each frame from the
# webcam. To improve FPS, the webcam object runs in a separate thread from the main program.
# This script will work with either a Picamera or regular USB webcam.
#
# This code is based off the TensorFlow Lite image classification example at:
# https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/examples/python/label_image.py
#
# I added my own method of drawing boxes and labels using OpenCV.
# Import packages
import os
import argparse
import cv2
import numpy as np
import sys
import time
from threading import Thread
import importlib.util
import simpleaudio as sa
# Define VideoStream class to handle streaming of video from webcam in separate processing thread
# Source - Adrian Rosebrock, PyImageSearch: https://www.pyimagesearch.com/2015/12/28/increasing-raspberry-pi-fps-with-python-and-opencv/
class VideoStream:
"""Camera object that controls video streaming from the Picamera"""
def __init__(self,resolution=(640,480),framerate=30):
# Initialize the PiCamera and the camera image stream
self.stream = cv2.VideoCapture(0)
ret = self.stream.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*'MJPG'))
ret = self.stream.set(3,resolution[0])
ret = self.stream.set(4,resolution[1])
# Read first frame from the stream
(self.grabbed, self.frame) = self.stream.read()
# Variable to control when the camera is stopped
self.stopped = False
def start(self):
# Start the thread that reads frames from the video stream
Thread(target=self.update,args=()).start()
return self
def update(self):
# Keep looping indefinitely until the thread is stopped
while True:
# If the camera is stopped, stop the thread
if self.stopped:
# Close camera resources
self.stream.release()
return
# Otherwise, grab the next frame from the stream
(self.grabbed, self.frame) = self.stream.read()
def read(self):
# Return the most recent frame
return self.frame
def stop(self):
# Indicate that the camera and thread should be stopped
self.stopped = True
# Define and parse input arguments
parser = argparse.ArgumentParser()
parser.add_argument('--modeldir', help='Folder the .tflite file is located in',
required=True)
parser.add_argument('--graph', help='Name of the .tflite file, if different than detect.tflite',
default='masktracker.tflite')
parser.add_argument('--labels', help='Name of the labelmap file, if different than labelmap.txt',
default='facelabelmap.txt')
parser.add_argument('--threshold', help='Minimum confidence threshold for displaying detected objects',
default=0.5)
parser.add_argument('--resolution', help='Desired webcam resolution in WxH. If the webcam does not support the resolution entered, errors may occur.',
default='640x480')
parser.add_argument('--edgetpu', help='Use Coral Edge TPU Accelerator to speed up detection',
action='store_true')
args = parser.parse_args()
MODEL_NAME = args.modeldir
GRAPH_NAME = args.graph
LABELMAP_NAME = args.labels
min_conf_threshold = float(args.threshold)
resW, resH = args.resolution.split('x')
imW, imH = int(resW), int(resH)
use_TPU = args.edgetpu
# Import TensorFlow libraries
# If tflite_runtime is installed, import interpreter from tflite_runtime, else import from regular tensorflow
# If using Coral Edge TPU, import the load_delegate library
pkg = importlib.util.find_spec('tflite_runtime')
if pkg:
from tflite_runtime.interpreter import Interpreter
if use_TPU:
from tflite_runtime.interpreter import load_delegate
else:
from tensorflow.lite.python.interpreter import Interpreter
if use_TPU:
from tensorflow.lite.python.interpreter import load_delegate
# If using Edge TPU, assign filename for Edge TPU model
if use_TPU:
# If user has specified the name of the .tflite file, use that name, otherwise use default 'edgetpu.tflite'
if (GRAPH_NAME == 'masktracker.tflite'):
GRAPH_NAME = 'edgetpu.tflite'
# Get path to current working directory
CWD_PATH = os.getcwd()
# Path to .tflite file, which contains the model that is used for object detection
PATH_TO_CKPT = os.path.join(CWD_PATH,MODEL_NAME,GRAPH_NAME)
# Path to label map file
PATH_TO_LABELS = os.path.join(CWD_PATH,MODEL_NAME,LABELMAP_NAME)
# Load the label map
with open(PATH_TO_LABELS, 'r') as f:
labels = [line.strip() for line in f.readlines()]
# Have to do a weird fix for label map if using the COCO "starter model" from
# https://www.tensorflow.org/lite/models/object_detection/overview
# First label is '???', which has to be removed.
if labels[0] == '???':
del(labels[0])
# Load the Tensorflow Lite model.
# If using Edge TPU, use special load_delegate argument
if use_TPU:
interpreter = Interpreter(model_path=PATH_TO_CKPT,
experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
print(PATH_TO_CKPT)
else:
interpreter = Interpreter(model_path=PATH_TO_CKPT)
interpreter.allocate_tensors()
# Get model details
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
height = input_details[0]['shape'][1]
width = input_details[0]['shape'][2]
floating_model = (input_details[0]['dtype'] == np.float32)
input_mean = 127.5
input_std = 127.5
# Initialize frame rate calculation
frame_rate_calc = 1
freq = cv2.getTickFrequency()
global image_capture
image_capture = 0
# Initialize video stream
videostream = VideoStream(resolution=(imW,imH),framerate=30).start()
time.sleep(1)
#for frame1 in camera.capture_continuous(rawCapture, format="bgr",use_video_port=True):
while True:
# Start timer (for calculating frame rate)
t1 = cv2.getTickCount()
# Grab frame from video stream
frame1 = videostream.read()
# Acquire frame and resize to expected shape [1xHxWx3]
frame = frame1.copy()
frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
frame_resized = cv2.resize(frame_rgb, (width, height))
input_data = np.expand_dims(frame_resized, axis=0)
# Normalize pixel values if using a floating model (i.e. if model is non-quantized)
if floating_model:
input_data = (np.float32(input_data) - input_mean) / input_std
# Perform the actual detection by running the model with the image as input
interpreter.set_tensor(input_details[0]['index'],input_data)
interpreter.invoke()
# Retrieve detection results
boxes = interpreter.get_tensor(output_details[0]['index'])[0] # Bounding box coordinates of detected objects
classes = interpreter.get_tensor(output_details[1]['index'])[0] # Class index of detected objects
scores = interpreter.get_tensor(output_details[2]['index'])[0] # Confidence of detected objects
#num = interpreter.get_tensor(output_details[3]['index'])[0] # Total number of detected objects (inaccurate and not needed)
# Loop over all detections and draw detection box if confidence is above minimum threshold
for i in range(len(scores)):
if ((scores[i] > min_conf_threshold) and (scores[i] <= 1.0)):
# Get bounding box coordinates and draw box
# Interpreter can return coordinates that are outside of image dimensions, need to force them to be within image using max() and min()
ymin = int(max(1,(boxes[i][0] * imH)))
xmin = int(max(1,(boxes[i][1] * imW)))
ymax = int(min(imH,(boxes[i][2] * imH)))
xmax = int(min(imW,(boxes[i][3] * imW)))
# Draw label
object_name = labels[int(classes[i])] # Look up object name from "labels" array using class index
if (object_name=="face unclear" ):
color = (0, 255, 0)
cv2.rectangle(frame, (xmin,ymin), (xmax,ymax),color, 2)
print("Face Covered: Door Not Opened")
if(image_capture == 0):
path = r'/home/pi/Desktop/tflite_1/photographs'
date_string = time.strftime("%Y-%m-%d_%H%M%S")
#print(date_string)
cv2.imwrite(os.path.join(path, (date_string + ".jpg")),frame)
#cv2.imshow("Photograph",frame)
#mp3File = input(alert_audio.mp3)
print("Photo Taken")
#w_object = sa.WaveObject.from_wave_file('alert_audio.wav')
#p_object = w_object.play()
#p_object.wait_done()
image_capture = 1
else:
color = (0, 0, 255)
cv2.rectangle(frame, (xmin,ymin), (xmax,ymax),color, 2)
print("Face Clear: Door Opened")
image_capture = 0
#cv2.rectangle(frame, (xmin,ymin), (xmax,ymax),color, 2)
#image = np.asarray(ImageGrab.grab())
label = '%s: %d%%' % (object_name, int(scores[i]*100)) # Example: 'person: 72%'
labelSize, baseLine = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2) # Get font size
label_ymin = max(ymin, labelSize[1] + 10) # Make sure not to draw label too close to top of window
cv2.rectangle(frame, (xmin, label_ymin-labelSize[1]-10), (xmin+labelSize[0], label_ymin+baseLine-10), (255, 255, 255), cv2.FILLED) # Draw white box to put label text in
cv2.putText(frame, label, (xmin, label_ymin-7), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0), 2) # Draw label text
if ((scores[0] < min_conf_threshold)):
cv2.putText(frame,"No Face Detected",(260,260),
cv2.FONT_HERSHEY_SIMPLEX, 0.8, color=(255,0,0))
print("No Face Detected")
image_capture = 0
# Draw framerate in corner of frame
cv2.putText(frame,'FPS: {0:.2f}'.format(frame_rate_calc),(30,50),cv2.FONT_HERSHEY_SIMPLEX,1,(255,255,0),2,cv2.LINE_AA)
# All the results have been drawn on the frame, so it's time to display it.
cv2.imshow('Object detector', frame)
# Calculate framerate
t2 = cv2.getTickCount()
time1 = (t2-t1)/freq
frame_rate_calc= 1/time1
# Press 'q' to quit
if cv2.waitKey(1) == ord('q'):
break
# Clean up
cv2.destroyAllWindows()
videostream.stop()
Any help is appreciated.
Regards,
MD
My printer Epson_L1110_Series I set in CUPS. It supports color print and now works only color mode.I am using "printFile() function for printing and I should print in monochrome mode.Thank you for help.
import cups
conn = cups.Connection()
file_name = '/home/pi/Downloads/File.pdf'
conn.printFile("EPSON_L1110_Series", file_name, "Pi Report",{"cpi": "6","lpi":"6","orientation-requested":"3", "print-color-mode":"monochrome"})
but it is not working.Printer still printed with color.
Hi i want to overlay or paste an image on bigger images(have a folder containinf 10 images and want to overlay the smaller images on all 10) and save them in a different folder. I did try somethings but ran into errors.
import scipy.misc
import numpy as np
import os
import cv2
outPath = "C:\darkflow\Augmented Images\augmented_images\.."
cov = cv2.imread("C:\darkflow\Augmented Images\extracted\cover\extracted_cover.jpg")
bgs = [cv2.imread(file) for file in glob.glob("C:\darkflow\Augmented Images\images\*.jpg")]
for bg in bgs:
bg[y_offset:y_offset+s_img.shape[0], x_offset:x_offset+s_img.shape[1]] = cov
f_image = cv2.cvtColor(bg, cv2.COLOR_BGR2RGB)
fullpath = os.path.join(outPath, 'augmented_'+ bg)
misc.imsave(fullpath, f_image)
with this code i get an error : ufunc 'add' did not contain a loop with signature matching types dtype('
I found the answer while looking into the code. My code is
from scipy import ndimage, misc
import scipy.misc
import numpy as np
import os
import cv2
cov = cv2.imread("C:\darkflow\Augmented Images\extracted\cover\extracted_cover.jpg")
bgs = [cv2.imread(file) for file in glob.glob("C:\darkflow\Augmented Images\images\*.jpg")]
d=1
x_offset=100
y_offset= 100
for bg in bgs:
bg[y_offset:y_offset+ cov.shape[0], x_offset:x_offset+ cov.shape[1]] = cov
filename = "images/file_%d.jpg"%d
cv2.imwrite(filename, bg)
d+=1
I've trained darkflow on my data set and have good result! I can feed it a pre recorded image or video and it draws the bounding boxes around the right things, win!
Now I'd like to run it live as has been done with camera feeds, except I'd like my feed to be from the screen, not the camera. I have a specific window, which is launched from a specific process, or I can just take a section of the screen (from coords) either is fine for my application.
Currently I use PILs image grab and then feed the images into darkflow, but this feels quite slow (maybe a few frames per second) nothing like the 30 ish fps you can get with video files!
I get more than 25 fps with Python MSS on my slow laptop under Ubuntu.
Here is an example:
from mss import mss
from PIL import Image
import time
def capture_screenshot():
with mss() as sct:
monitor = sct.monitors[1]
sct_img = sct.grab(monitor)
# Convert to PIL/Pillow Image
return Image.frombytes('RGB', sct_img.size, sct_img.bgra, 'raw', 'BGRX')
N = 100
t = time.time()
for _ in range(N):
capture_screenshot()
print ("Frame rate = %.2f fps" % (N/(time.time()-t)))
Output:
Frame rate = 27.55 fps
I got over 40 fps with this script (on i5-7500 3.4GHz, GTX 1060, 48GB RAM).
There are a lot of APIs used to capture the screen. Among them, mss runs much faster and is not difficult to use. Here is an implementation of mss with darkflow(YOLOv2), in which 'mon' defines the area you want apply prediction on the screen.
options is passed to the darkflow, that specifies which config file and checkpoint we want to use, threshold for detection, and how much this process occupies the GPU. Before we run this script, we have to have at least one trained model (or Tensorflow checkpoint). Here, load is the checkpoint number.
If you think that the network detects too many bounding boxes, I recommend you to lower the threshold.
import numpy as np
import cv2
import glob
from moviepy.editor import VideoFileClip
from mss import mss
from PIL import Image
from darkflow.net.build import TFNet
import time
options = {
'model' : 'cfg/tiny-yolo-voc-1c.cfg' ,
'load' : 5500,
'threshold' : 0.1,
'gpu' : 0.7 }
tfnet = TFNet( options )
color = (0, 255, 0) # bounding box color.
# This defines the area on the screen.
mon = {'top' : 10, 'left' : 10, 'width' : 1000, 'height' : 800}
sct = mss()
previous_time = 0
while True :
sct.get_pixels(mon)
frame = Image.frombytes( 'RGB', (sct.width, sct.height), sct.image )
frame = np.array(frame)
# image = image[ ::2, ::2, : ] # can be used to downgrade the input
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
results = tfnet.return_predict( frame )
for result in results :
tl = ( result['topleft']['x'], result['topleft']['y'] )
br = ( result['bottomright']['x'], result['bottomright']['y'] )
label = result['label']
confidence = result['confidence']
text = '{} : {:.0f}%'.format( label, confidence * 100 )
frame = cv2.rectangle( frame, tl, br, color, 5 )
frame = cv2.putText( frame, text, tl, cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 0), 2 )
cv2.imshow ( 'frame', frame )
if cv2.waitKey ( 1 ) & 0xff == ord( 'q' ) :
cv2.destroyAllWindows()
txt1 = 'fps: %.1f' % ( 1./( time.time() - previous_time ))
previous_time = time.time()
print txt1
In a given DataSeries I want to use a Bokeh slider to change the number of observations I show (the tail). Can this be done, if it can why won't my script work?
import pandas as pd
from bokeh.plotting import show, output_file
from bokeh.models import CustomJS, Slider, Column
from bokeh.io import output_notebook
ds = pd.Series( [i for i in range(20)], pd.date_range('2016-01-02', periods=20, freq='D'))
tail = 5
ds.tail(tail)
#Bokeh slider to change value of "tail"
s1 = Slider(start=1, end=20, value=3, step=1)
s1.callback = CustomJS(args=dict(s1=s1, tail=tail), code="""
ds.tail("tail", s1.get('value'));
""")
show(Column(s1))
ds.tail(tail)