Finger peak detection using MATLAB - matlab

Question

I have to create an algorithm with Matlab that, with a image of a hand, can know the form of the hand by the number of raised fingers and the presence or absence of the thumb. So far, the algorithm is almost complete but I don't know what more I can do that could find the peaks that represents the fingers. We tried a lot of things but nothing works. The idea is to find when there is a sudden increasement but as the pixels are never completely aligned, nothing that we tried worked. Someone has any idea? Here is the code so far.
The image that he is reading is this one:
To know if the finger is relevant or not, we already have an idea that might work... but we need to find the fingers first.
clear all
close all
image=imread('mao2.jpg');
YCBCR = rgb2ycbcr(image);
image=YCBCR;
cb = image(:,:,2);
cr = image(:,:,3);
imagek(:,1) = cb(:);
imagek(:,2) = cr(:);
imagek = double(imagek);
[IDX, C] = kmeans(imagek, 2, 'EmptyAction', 'singleton');
s=size(image);
IDX= uint8(IDX);
C2=round(C);
imageNew = zeros(s(1),s(2));
temp = reshape(IDX, [s(1) s(2)]);
for i = 1 : 1 : s(1)
for j = 1 : 1 : s(2)
imageNew(i,j,:) = C2(temp(i,j));
end
end
imageNew=uint8(imageNew);
[m,n]=size(imageNew);
for i=1:1:m
for j = 1:1:n
if(imageNew(i,j)>=127)
pretobranco(i,j)=0;
else
pretobranco(i,j)=1;
end
end
end
I2=imfill(pretobranco);
imshow(I2);
imwrite(I2, 'mao1trab.jpg');
[m,n]=size(I2);
B=edge(I2);
figure
imshow(B);
hold on;
stats=regionprops(I2,'BoundingBox');
rect=rectangle('position', [stats(1).BoundingBox(1), stats(1).BoundingBox(2), stats(1).BoundingBox(3), stats(1).BoundingBox(4)], 'EdgeColor', 'r');
stats(1).BoundingBox(1)
stats(1).BoundingBox(2)
stats(1).BoundingBox(3)
stats(1).BoundingBox(4)
figure
Bound = B( stats(1).BoundingBox(2): stats(1).BoundingBox(2)+stats(1).BoundingBox(4)-1, stats(1).BoundingBox(1):stats(1).BoundingBox(1)+stats(1).BoundingBox(3)-1);
imshow(Bound)
y1 = round(stats(1).BoundingBox(2))
y2 = round(stats(1).BoundingBox(2)+stats(1).BoundingBox(4)-1)
x1 = round(stats(1).BoundingBox(1))
x2 = round(stats(1).BoundingBox(1)+stats(1).BoundingBox(3)-1)
% Bounding box contida em imagem[M, N].
[M,N] = size(Bound)
vertical=0;
horizontal=0;
if M > N
vertical = 1 %imagem vertical
else
horizontal = 1 %imagem horizontal
end
%Find thumb
MaoLeft = 0;
MaoRight = 0;
nPixelsBrancos = 0;
if vertical==1
for i = x1:1:x2
for j= y1:1:y2
if I2(j,i) == 1
nPixelsBrancos = nPixelsBrancos + 1; %Numero de pixels da mão
end
end
end
for i=x1:1:x1+30
for j=y1:1:y2
if I2(j,i) == 1
MaoLeft = MaoLeft + 1; %Number of pixels of the hand between the 30 first colums
end
end
end
for i=x2-30:1:x2
for j=y1:1:y2
if I2(j,1) == 1
MaoRight = MaoRight + 1; %Number of pixels of the hand between the 30 last colums
end
end
end
TaxaBrancoLeft = MaoLeft/nPixelsBrancos
TaxaBrancoRight = MaoRight/nPixelsBrancos
if TaxaBrancoLeft <= (7/100)
if TaxaBrancoRight <= (7/100)
Thumb = 0 %Thumb in both borders is defined as no Thumb.
else
ThumbEsquerdo = 1 %Thumb on left
end
end
if TaxaBrancoRight <= (7/100) && TaxaBrancoLeft >= (7/100)
ThumbDireito = 1 %Thumb on right
end
end
if horizontal==1
for i = x1:1:x2
for j= y1:y2
if I2(i,j) == 1
nPixelsBrancos = nPixelsBrancos + 1; %Numero de pixels da mão
end
end
end
for i=x1:1:x2
for j=y1:1:y1+30
if I2(i,j) == 1
MaoLeft = MaoLeft + 1; %Numero de pixels da mão entre as 30 primeiras colunas
end
end
end
for i=x1:1:x2
for j=y2-30:1:y2
if I2(j,1) == 1
MaoRight = MaoRight + 1; %Numero de pixels da mão entre as 30 ultimas colunas
end
end
end
TaxaBrancoLeft = MaoLeft/nPixelsBrancos
TaxaBrancoRight = MaoRight/nPixelsBrancos
if TaxaBrancoLeft <= (7/100)
if TaxaBrancoRight <= (7/100)
Thumb = 0 %Polegar nas duas bordas. Definimos como sem polegar.
else
ThumbEsquerdo = 1 %Polegar na borda esquerda
end
end
if TaxaBrancoRight <= (7/100) && TaxaBrancoLeft >= (7/100)
ThumbDireito = 1 %Polegar na borda direita
end
end
figure
imshow(I2);
%detecção da centroid
Ibw = im2bw(I2);
Ilabel = bwlabel(Ibw);
stat = regionprops(Ilabel,'centroid');
figure
imshow(I2); hold on;
for x = 1: numel(stat)
plot(stat(x).Centroid(1),stat(x).Centroid(2),'ro');
end
centroid = [stat(x).Centroid(1) stat(x).Centroid(2)] %coordenadas x e y da centroid
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

Answer 1

Seemed like an interesting problem, so I gave it a shot. Basically you start with a Sobel filter to find the edges in your image (after slight denoising). Then clean up the resulting lines, use them to separate regions within your binary mask of the hand, use a watershed transform to find the wrist, some distance transforms to find other landmarks, then remove the palm. What you're left with is separate regions for each finger and thumb. You can count those regions easily enough or find which way they are pointing, or whatever you'd like.
imgURL = 'https://encrypted-tbn2.gstatic.com/imgs?q=tbn:ANd9GcRQsqJtlrOnSbJNTnj35Z0uG9BXsecX2AXn1vV0YDKodq-zSuqnnQ';
imgIn=imread(imgURL);
gaussfilt = fspecial('gaussian', 3, .5); % Blur starting image
blurImg = imfilter(double(img(:,:,1)), gaussfilt);
edgeImg = edge(blurImg, 'sobel'); % Use Sobel edge filter to pick out contours of hand + fingers
% Clean up contours
edgeImg = bwmorph(edgeImg, 'close', 1);
edgeImg = bwmorph(edgeImg, 'thin', Inf);
% Clean up rogue spots in corners
edgeImg([2 end-1], 2) = 0;
edgeImg([2 end-1], end-1) = 0;
% Extend lines to edge of image (correct for 'close' operation above
edgeImg([1 end],:) = edgeImg([2 end-1],:);
edgeImg(:, [1 end]) = edgeImg(:, [2 end-1]);
% Remove all but the longest line
regs = regionprops(edgeImg, 'Area', 'PixelIdxList');
regs(vertcat(regs.Area) ~= max(vertcat(regs.Area))) = [];
lineImg = false(size(edgeImg, 1), size(edgeImg, 2));
lineImg(regs.PixelIdxList) = 1;
fillImg = edgeImg;
% Close in wrist
if any(fillImg(1,:))
fillImg(1,:) = 1;
end
if any(fillImg(end,:))
fillImg(end,:) = 1;
end
if any(fillImg(:,1))
fillImg(:,1) = 1;
end
if any(fillImg(:,end))
fillImg(:,end) = 1;
end
fillImg = imfill(fillImg, 'holes');
fillImg([1 end], :) = 0;
fillImg(:, [1 end]) = 0;
fillImg([1 end],:) = fillImg([2 end-1],:);
fillImg(:, [1 end]) = fillImg(:, [2 end-1]);
% Start segmenting out hand + fingers
handBin = fillImg;
% Set lines in above image to 0 to separate closely-spaced fingers
handBin(lineImg) = 0;
% Erode these lines to make fingers a bit more separate
handBin = bwmorph(handBin, 'erode', 1);
% Segment out just hand (remove wrist)
distImg = bwdist(~handBin);
[cDx, cDy] = find(distImg == max(distImg(:)));
midWrist = distImg;
midWrist = max(midWrist(:)) - midWrist;
midWrist(distImg == 0) = Inf;
wristWatershed = watershed(imerode(midWrist, strel('disk', 10)));
whichRegion = wristWatershed(cDx, cDy);
handBin(wristWatershed ~= whichRegion) = 0;
regs = regionprops(handBin, 'Area', 'PixelIdxList');
regs(vertcat(regs.Area) ~= max(vertcat(regs.Area))) = [];
handOnly = zeros(size(handBin, 1), size(handBin, 2));
handOnly(regs.PixelIdxList) = 1;
% Find radius of circle around palm centroid that excludes wrist and splits
% fingers into separate regions.
% This is estimated as D = 1/3 * [(Centroid->Fingertip) + 2*(Centroid->Wrist)]
% Find Centroid-> Wrist distance
dist2w = wristWatershed ~= whichRegion;
dist2w = bwdist(dist2w);
distToWrist = dist2w(cDx, cDy);
% Find Centroid-> Fingertip distance
dist2FE = zeros(size(handOnly, 1), size(handOnly, 2));
dist2FE(cDx, cDy) = 1;
dist2FE = bwdist(dist2FE).*handOnly;
distToFingerEnd = max(dist2FE(:));
circRad = mean([distToFingerEnd, distToWrist, distToWrist]); % Estimage circle diameter
% Draw circle
X = bsxfun(#plus,(1:size(handOnly, 1))',zeros(1,size(handOnly, 2)));
Y = bsxfun(#plus,(1:size(handOnly, 2)),zeros(size(handOnly, 1),1));
B = sqrt(sum(bsxfun(#minus,cat(3,X,Y),reshape([cDx, cDy],1,1,[])).^2,3))<=circRad;
% Cut out binary mask within circle
handOnly(B) = 0;
% Label separate regions, where each now corresponds to a separate digit
fingerCount = bwlabel(handOnly);
% Display overlay image
figure()
imshow(imgIn)
hold on
overlayImg = imshow(label2rgb(fingerCount, 'jet', 'k'));
set(overlayImg, 'AlphaData', 0.5);
hold off
Results:
http://imgur.com/ySn1fPy