基于单 camera的手势识别

android平台测试程序见附件。
http://pan.baidu.com/s/1gd5nthL
密码： h46g

基于视觉的动作识别，一直以来的最大问题是精度问题。 - -！

刚刚做了一套手势识别的算法，在此做下简单总结

先说下我做的效果：

2m内手掌识别率在90%以上（强光干扰下效果烂成渣渣了。。so。。 扣掉10%）

处理速度，15fps

cpu 30% （arm 4核）

以上效果为平板测试结果

1> 目的： opencv的haar特征库用来训练各种手势其实是很强势，很好用的东西，唯一的缺点是： Haar版权问题。。，

Haar训练需要的样本数也是一个蛮头疼的问题，识别准确性完全取决于样本数量和质量，没有几十k的样本，效果只能呵呵了

还好，条条大路通罗马，

2>外围设备： 摄像头，这是基本的

深度传感器 ，这是一直想要的，可惜到现在也没搞到

红外传感器 ，同上。。。。。

3> 检测方式： 利用手势的特征点：

手掌的特征点还是挺多的，我使用的是手轮廓的5个内凹陷，再加上相对位置，再加上人体肤色的特征

基本一个完整的手就检测出来了

4> 缺点： 精度缺失 ，没办法识别3d空间位置，单camera。。。。

--------------------------------code 分割线--------------------------------------------------------

肤色过滤部分

#include "utils.h"

cv::Mat edgeFilter();
void areaFilter(cv::Mat srcMask);
void skinFilter();

cv::Scalar YUV_SKIN_BEGIN = cv::Scalar(0,133,77);   // 论文肤色(0,133,77)->(256,173,127)
cv::Scalar YUV_SKIN_END   = cv::Scalar(256,173,127);

cv::Scalar COLOR_RED = cv::Scalar(0,0,255);
cv::Scalar COLOR_GREEN = cv::Scalar(0,255,0);
cv::Scalar COLOR_BLUE = cv::Scalar(255,0,0);

#ifdef USE_SHARP
cv::Mat sharpKernal = (cv::Mat_<float>(3, 3) << 0, -1, 0, -1, 5, -1, 0, -1, 0);
#endif

cv::Mat morphKernal = cv::getStructuringElement(cv::MORPH_RECT,cv::Size(3,3),cv::Point(1,1) );

cv::Size pSize = cv::Size(320,240);
//  边缘轮廓
cv::Mat edgeFilter(){
#ifdef USE_EDGE
#ifdef DEBUG
long begT = getTime(true);
#endif
cv::Mat g = gray.clone();

if(withRsize)
resize(g,g,pSize);
equalizeHist(g,g);
Canny(g,g,100,300,3);
g = g > 1;

#ifdef USE_SHARP
long usec = getTime(true);
filter2D(gray, gray, gray.depth(), sharpKernal); //卷积
LOGD("filter2D(%d*%d): %d us",gray.cols,gray.rows,(int)(getTime(true)-usec));
#endif

#ifdef USE_ERODE
dilate(gray,gray,morphKernal);
#endif

if(withRsize)
resize(g,g,src.size());

#ifdef DEBUG
int w = withRsize? 320 : gray.cols;
int h = withRsize? 240 : gray.rows;
cv::Mat can;
cvtColor(g,can,CV_GRAY2BGR);
//src.copyTo(can,g);
//addWeighted(src,1.0,can,1.0,1.0,src);
src = src + can;
LOGD("edgeFilter(%d*%d): %d us",w,h,(int)(getTime(true)-begT));
#endif
return g;
#endif //USE_EDGE
}

// 小面积过滤
void areaFilter(cv::Mat srcMask){
#ifdef USE_SF_AREA
#ifdef DEBUG
long usec = getTime(true);
#endif
cv::Mat temp = srcMask.clone(); // 二值化图像
std::vector<std::vector<cv::Point> > points;
std::vector<cv::Vec4i> vecs;
findContours(temp, points, vecs,
CV_RETR_EXTERNAL,  CV_CHAIN_APPROX_SIMPLE );

if(points.size()<=0 || vecs.size()<=0) {
skin.setTo(0);
return;
}

std::vector<std::vector<cv::Point> > points_poly(points.size());
std::vector<cv::Rect> bound(points.size());
std::vector<int> usefulList;
std::vector<float> areaList;
float max=0;
for(int i=0;i>=0;i=vecs[i][0]){
bound[i] = boundingRect(points[i]);
//噪声过滤（微小面积过滤）
if( points[i].size()<6
|| bound[i].width < MIN_IMAGE_PIX || bound[i].height < MIN_IMAGE_PIX
) {
srcMask(bound[i]).setTo(0);
continue;
}else{ // 多边形逼近
approxPolyDP(points[i], points_poly[i],10,true);
float area = fabs(contourArea(points_poly[i]));
// 有用Rect
usefulList.push_back(i);  // 索引
areaList.push_back(area); // 面积
if(max<area) max = area;  // 最大面积
}
}
max *= 0.01;
int j=usefulList.size()-1;

for(int i=0;j>=0;j--){
i = usefulList.at(j);
float area = areaList.at(j);
//过滤
if(area<=max) {
srcMask(bound[i]).setTo(0);
continue;
}
#ifdef USE_ERODE
std::vector<int> hull;
std::vector<cv::Point> pos = points_poly[i];
convexHull(cv::Mat(pos), hull, true);
try{
std::vector<cv::Vec4i> defects; // 起始点，终止点，最凹点，凹点深度
convexityDefects(pos,cv::Mat(hull),defects);
int dsize = defects.size();
if(dsize<=0) continue;
int miniDis = MAX(bound[i].width,bound[i].height)>>3;

for(int j=0;j<dsize;j++){
if(miniDis>(defects[j][3]>>8)) continue;
cv::Rect rect = cv::Rect(pos[defects[j][2]].x-(miniDis>>1)
,pos[defects[j][2]].y-(miniDis>>1)
,(miniDis)
,(miniDis));
cv::Mat roi = srcMask(rect);
cv::Mat kernel = getStructuringElement(cv::MORPH_RECT,cv::Size(3,(miniDis>>1)+1),cv::Point(1,(miniDis>>2)+1) );
erode(roi,roi,kernel);
medianBlur(roi,roi,3);
}

}catch(cv::Exception e){}
#endif
}
#ifdef DEBUG
LOGD("areaFilter(%d*%d): %d us",srcMask.cols,srcMask.rows,(int)(getTime(true)-usec));
#endif
#endif
}

cv::Mat skin_img;
cv::Mat cany;
void *edgeFilterT(void*){
skin_img = edgeFilter();
}

void skinFilter(){
#ifdef DEBUG
long usec = getTime(true);
LOGD("skinFilter()");
#endif

#ifdef USE_EDGE
int err;
pthread_t tid;
err = pthread_create(&tid,NULL,&edgeFilterT,NULL);
if(err!=0){
skin_img = edgeFilter();
}
#endif

cv::Mat tmpColor;
cv::cvtColor(src,tmpColor,CV_BGR2YCrCb);
cv::Mat skinMask = cv::Mat::zeros(tmpColor.size(),CV_8UC1);
cv::inRange(tmpColor,YUV_SKIN_BEGIN,YUV_SKIN_END,skinMask);

#ifdef USE_EDGE
if(err==0){
err = pthread_join(tid,NULL);
}
skinMask -= skin_img;
cany = skin_img;
#endif
skinMask -= gray;

#ifdef USE_SF_AREA
areaFilter(skinMask);
#endif

#ifdef DEBUG
LOGD("skinFilter:BGR->YUV:(%d,%d) : %d us",src.cols,src.rows,(int)(getTime(true)-usec));
#endif
mask = skinMask;    // yuv uv分量肤色mask
}

手掌识别部分：

#include "utils.h"

// (0 - 3)   0:最准确  3:误判最多
#define PLAM_THRESHOLD 0
bool isPlamFound = false;
bool withPlam = true;

void actionPlam(RectHold found);
void actionFist(cv::Point center);
void findArea(); // 面积处理
bool findPlam(cv::Mat mask,cv::Rect roi);
int getDistance(cv::Point p1,cv::Point p2,bool usemax=false);
//void findPlam(std::vector<UsefulContours> uContours);  // 计算轮廓

int getDistance(cv::Point p1,cv::Point p2,bool usemax) {
int x = abs(p1.x-p2.x);
int y = abs(p1.y-p2.y);
return usemax ? MAX(x,y) : sqrt(x*x+y*y);
}

void actionPlam(RectHold found){
//#ifdef DEBUG
if(showRect)
rectangle(src,found.bound.tl(),found.bound.br(),COLOR_RED,3,8,0);
LOGD("found plam (%d,%d) ,%d",found.center.x,found.center.y,found.c);
//#endif
cv::Rect pRect = found.bound;
//lastPlamHold.lastRect = cv::Rect(pRect.tl(),pRect.br());
lastPlamHold.lastRect = found.bound;
lastPlamHold.center = found.center;
lastPlamHold.area = found.area;
lastPlamHold.arc = found.arc;
// lastPlamHold.centRect = cv::Rect(found.center.x-(pRect.width>>2)
//        ,found.center.y-(pRect.height>>2) ,pRect.width>>1 ,pRect.height>>1);
lastPlamHold.centRect = found.cRect;
if(!lastPlamHold.used){
lastPlamHold.used = true;
lastPlamHold.unHoldCount=0;
return;
}
#ifdef USE_OPT_FLOW
optInit = true;

#ifndef OPT_NEED_INIT
withPlam = false;
#endif

#endif
// 关闭camshift
#ifdef USE_CAMSHIFT
#ifdef USE_FIST
trackRect = false;
initBackproj = false;
#else
trackRect = true;
initBackproj = true;
#endif // USE_FIST
#endif // USE_CAMSHIFT
/////////////////
lastPlamHold.type = TYPE_PLAM;
callJava(lastPlamHold.type,lastPlamHold.center);
}

void actionFist(cv::Point center){
lastPlamHold.used = true;
lastPlamHold.unHoldCount=0;
lastPlamHold.center = center;

// 关闭camshift
#ifdef USE_CAMSHIFT
trackRect = true;
initBackproj = true;
#endif // USE_CAMSHIFT
/////////////////
lastPlamHold.type = TYPE_FIST;
callJava(lastPlamHold.type,lastPlamHold.center);
}

cv::Mat plam_img;
cv::Rect plam_rect;
pthread_mutex_t locker = PTHREAD_MUTEX_INITIALIZER;
void *findPlamT(void*){
findPlam(plam_img,plam_rect);
}

//   截取不同区域计算手掌
long pTime = 0;
std::vector<RectHold> pHolds;
bool findPlam(cv::Mat mask,cv::Rect roi){
#ifdef DEBUG
long beginT = getTime(true);
#endif
cv::Point sp = cv::Point(roi.x,roi.y);
cv::Rect   bound_find;
std::vector<cv::Point>  contours_find;
std::vector<cv::Point>  contours_poly_find;
std::vector<RectHold> plamHold;
float max = -1;
float arc = -1;
{
int width = roi.width >> 1;
int height = roi.height >> 1;
bool wLarge = width >= height;
int mMax = MAX(roi.width,roi.height);
int mMin = MIN(roi.width,roi.height);
if(mMin<=8) return false;
int sv = mMax/mMin;
if(sv>1 && mMax>30){
//if(false) {
// width : height not normal split it
cv::Rect roiRect1,roiRect2;  // 截取两端
int x,y,w,h;
if(wLarge){  // (0,0,1.5w,1.5h)
// for roiRect1
w = roi.height + height - (height>>2);
h = roi.height;
// for roiRect2
x = roi.x+roi.width-w;
y = roi.y;

}else{
w = roi.width;
h = roi.width + width - (width>>2);
x = roi.x;
y = roi.y+roi.height-h;
}

roiRect1 = cv::Rect(roi.x,roi.y,w,h) & cv::Rect(0,0,mask.cols,mask.rows);
roiRect2 = cv::Rect(x,y,w,h) & cv::Rect(0,0,mask.cols,mask.rows);

plam_img = mask;
plam_rect = roiRect1;
#ifdef USE_THREAD
pthread_t tid;
pthread_create(&tid,NULL,&findPlamT,NULL);
#else
findPlam(mask,roiRect1);// recursive
#endif
findPlam(mask,roiRect2);// recursive
#ifdef USE_THREAD
pthread_join(tid,NULL);
#endif
return false;
}
cv::Mat tmp = mask(roi).clone();
std::vector<std::vector<cv::Point> > contours;
std::vector<cv::Vec4i> vecs;
findContours(tmp,contours,vecs,CV_RETR_EXTERNAL,CV_CHAIN_APPROX_SIMPLE);

if(contours.size()<=0 || vecs.size()<=0)    return false;

std::vector<std::vector<cv::Point> > contours_poly(contours.size());
std::vector<cv::Rect> bound(contours.size());

for(int i=0;i>=0;i=vecs[i][0]){
bound[i] = boundingRect(contours[i]);
if(contours[i].size()>6
&& bound[i].width>width
&& bound[i].height>height){
approxPolyDP(contours[i],contours_poly[i],5.0,true);
if(contours_poly[i].size()<=3) continue;
float area = fabs(contourArea(contours_poly[i]));
if(area>max){
arc = arcLength(contours_poly[i],true);
max  = area;
bound_find = bound[i];
contours_find = contours[i];
contours_poly_find = contours_poly[i];
}
}
} // for end
}
if(contours_poly_find.size()<=3) return false;
cv::Rect shRect = cv::Rect(bound_find.x+roi.x,bound_find.y+roi.y,bound_find.width,bound_find.height);
// now we find the max area,let's find out the plam

//Moments m = moments(contours_find,false);
// 绘制 凸包
cv::Rect cRect = cv::Rect(bound_find.x + (bound_find.width>>2)
,bound_find.y + (bound_find.height>>2)
,bound_find.width>>1
,bound_find.height>>1);

cv::Point center = cv::Point(cRect.x+(cRect.width>>1),cRect.y+(cRect.height>>1))+sp;
/*
if(lastPlamHold.used && lastPlamHold.centRect.contains(center)){
rectangle(src,cRect.tl()+sp,cRect.br()+sp,COLOR_RED,3,8,0);
isPlamFound = true;
}
*/
//rectangle(src,cRect.tl()+sp,cRect.br()+sp,COLOR_RED,0.5,8,0);
std::vector<int> hull;
std::vector<cv::Point> pos = contours_poly_find;
convexHull(cv::Mat(pos), hull, true);
try{
std::vector<cv::Vec4i> defects; // 起始点，终止点，最凹点，凹点深度
convexityDefects(pos,cv::Mat(hull),defects);
int dsize = defects.size();
if(dsize<=0) return false;
// 凸凹陷点均值
int avg=0;
for(int j=0;j<dsize;j++)
avg += (defects[j][3] >>8);
avg /= dsize;
int miniDis = avg>>1;

cv::Point fixP = cv::Point(0,0); // 偏移修正量
int actSize = 0;
std::vector<int> pHold;
for(int j=0;j<dsize;j++){
if((defects[j][3]>>8) <= avg){
continue;
}
pHold.push_back(j);
cv::Point depP = pos[defects[j][2]]; //凸凹陷点
// cRect = 1/2 Bound.Rect
// 向落入cRect的极值点方向偏移，偏移量 fixP(x,y)
if(cRect.contains(pos[defects[j][2]])){ // 蓝色 一次修正
#ifdef DEBUG
if(showRect){
circle(src,pos[defects[j][2]]+sp,4,COLOR_BLUE,0.5,CV_AA);
line(src,pos[defects[j][1]]+sp,pos[defects[j][0]]+sp,COLOR_BLUE,2,CV_AA);
}
#endif
fixP.x += depP.x;
fixP.y += depP.y;
actSize ++;
}
#ifdef DEBUG
if(showRect){
cv::Vec4i vec = defects[j];
line(src,pos[vec[0]]+sp,pos[vec[2]]+sp,COLOR_RED,1.5,CV_AA);
line(src,pos[vec[1]]+sp,pos[vec[2]]+sp,COLOR_RED,2,CV_AA);
// farthest_pt_index
circle(src,pos[vec[2]]+sp,4,COLOR_RED,1,CV_AA);
circle(src,pos[vec[1]]+sp,4,COLOR_BLUE,2,CV_AA);
circle(src,pos[vec[0]]+sp,4,COLOR_GREEN,1,CV_AA);
} // end showRect
#endif
}//end for defects.size();
if(actSize<=1) {
#ifdef USE_NEW_FIST
// ==========================maybe fist
int areaBit = (int)lastPlamHold.area/max;

//int pMax = MAX(abs(lastPlamHold.center.x-center.x),abs(lastPlamHold.center.y-center.y));
if(lastPlamHold.used&&lastPlamHold.centRect.contains(center)
&& lastPlamHold.lastRect.width > shRect.width
&& lastPlamHold.lastRect.height > shRect.width
//&& pMax>2
&& areaBit>=0 && areaBit<3
//&& pHold.size()<=2
){

rectangle(src,shRect.tl(),shRect.br(),COLOR_BLUE,3,8,0);
actionFist(center);
}
if(lastPlamHold.unHoldCount++>fps) lastPlamHold.used=false;
initBackproj = true;
sRect = cRect & cv::Rect(0,0,src.cols,src.rows);
#endif
return false;   // fixP 偏移修正(x,y)
}
fixP.x /= actSize;
fixP.y /= actSize;
fixP.x -= (cRect.x + (cRect.width>>1));   // fixP 中心偏移
fixP.y -= (cRect.y + (cRect.height>>1));
// 修正后的rect
int lx = MIN(cRect.width,cRect.height)>>3;
cv::Rect actRect = cv::Rect(cRect.x+fixP.x-lx,cRect.y+fixP.y-lx
,cRect.width+(lx<<1),cRect.height+(lx<<1));
#ifdef DEBUG
rectangle(src,actRect.tl()+sp,actRect.br()+sp,COLOR_RED,0.5,8,0);
#endif

std::vector<int> tmpHold;
for(int i=pHold.size()-1;i>=0;i--){
cv::Point p = pos[defects[pHold[i]][2]];
if(actRect.contains(p))
tmpHold.push_back(pHold[i]);  // 生效点
}
// 判断极值点是否过近
for(int i=tmpHold.size()-1;i>0;i--){
cv::Point p1 = pos[defects[tmpHold[i]][2]];
cv::Point p2 = pos[defects[tmpHold[i-1]][2]];
int x = MAX(p1.x,p2.x)-MIN(p1.x,p2.x);
int y = MAX(p1.y,p2.y)-MIN(p1.y,p2.y);
if(MAX(x,y)<=(lx)) {
return false;
}
}

int actP=0; // 生效点个数
// 判断极值点是否与左右相连
dsize = tmpHold.size();
for(int i=0;i<dsize;i++){
int pl = tmpHold[(i+1)%dsize];
int p = tmpHold[i];
int pr = tmpHold[(i+dsize-1)%dsize];
int pdl = getDistance(pos[defects[p][0]],pos[defects[pl][1]],true);
int pdr = getDistance(pos[defects[p][1]],pos[defects[pr][0]],true);
if(pdl<miniDis || pdr < miniDis){
actP ++;
}
}

if(actP>=(3-PLAM_THRESHOLD)){
//rectangle(src,roi.tl(),roi.br(),COLOR_RED,3,8,0);
RectHold hold;
hold.bound = roi;
hold.cRect = actRect + sp;
hold.center.x = hold.cRect.x + (hold.cRect.width>>1);
hold.center.y = hold.cRect.y + (hold.cRect.height>>1);
hold.c = 1;
hold.area = max ;
hold.arc = arc ;
plamHold.push_back(hold);
}
}catch(cv::Exception e){}
if(plamHold.size()<=0) {
#ifdef DEBUG
LOGD("findPlam(%d*%d) (no): %d us",roi.width,roi.height,(int)(getTime(true)-beginT));
#endif
isPlamFound = false;
// viewpager ass this fist open
//        if(lastPlamHold.unHoldCount++>fps)
//        	lastPlamHold.used=false;
return false;
}
#ifdef USE_FPS_PLAM
actionPlam(plamHold[0]);
#else
isPlamFound = false;
//  多帧平均   初始化时间较长
long t = getTime();
if(pTime!=t){
LOGD(" time out....");
pTime = t;
std::vector<RectHold> tmpHold;
#ifdef USE_THREAD
pthread_mutex_lock(&locker);
#endif
for(int i=pHolds.size()-1;i>=0;i--){
if(pHolds[i].c>2)
tmpHold.push_back(pHolds[i]);
}
std::swap(pHolds,tmpHold);
#ifdef USE_THREAD
pthread_mutex_unlock(&locker);
#endif
}

if(pHolds.size()==0){
#ifdef USE_THREAD
pthread_mutex_lock(&locker);
#endif

std::swap(pHolds,plamHold);

#ifdef USE_THREAD
pthread_mutex_unlock(&locker);
#endif
return false;
}
for(int i=plamHold.size()-1;i>=0;i--){
bool unFind = true;
RectHold ph = plamHold[i];
#ifdef USE_THREAD
pthread_mutex_lock(&locker);
#endif
for(int j=pHolds.size()-1;j>=0;j--){
RectHold actHold = pHolds[j];
if(actHold.cRect.contains(ph.center)){
pHolds[j].c += ph.c;
unFind = false;
LOGD("__plam hold count: (%d)  fps:%d",pHolds[j].c,fps);
if(pHolds[j].c >= (fps>>2)){
//#ifdef DEBUG
rectangle(src,actHold.bound,COLOR_RED,3,8,0);
//#endif
isPlamFound = true;
actHold.c = (fps>>2)+2;
pHolds.clear();
pHolds.push_back(actHold);
actionPlam(actHold);
pthread_mutex_unlock(&locker);
return true;
}
}
}
if(unFind){
pHolds.push_back(ph);
}
#ifdef USE_THREAD
pthread_mutex_unlock(&locker);
#endif
}
#endif
#ifdef DEBUG
LOGD("findPlam(%d*%d): %d us",roi.width,roi.height,(int)(getTime(true)-beginT));
#endif
}

void findArea(){
#ifdef DEBUG
long beginT = getTime(true);
#endif
//std::vector<UsefulContours> usefulContoursList;
cv::Mat temp = mask.clone(); // 二值化图像
std::vector<std::vector<cv::Point> > points;
std::vector<cv::Vec4i> vecs;
findContours(temp, points, vecs,
CV_RETR_EXTERNAL,  CV_CHAIN_APPROX_SIMPLE );

if(points.size()<=0 || vecs.size()<=0) {
skin.setTo(0);
return;
}

std::vector<std::vector<cv::Point> > points_poly(points.size());
std::vector<cv::Rect> bound(points.size());
//std::vector<int> usefulList;
//std::vector<float> areaList;
for(int i=0;i>=0;i=vecs[i][0]){
bound[i] = boundingRect(points[i]);
if( points[i].size()<6) {
mask(bound[i]).setTo(0);
continue;
}else{ // 多边形逼近
#ifdef DEBUG
approxPolyDP(points[i], points_poly[i],10,true);
if(showRect) drawContours(src,points_poly,i,COLOR_GREEN);
#endif
/*
float area = fabs(contourArea(points_poly[i]));
// 有用Rect
usefulList.push_back(i);  // 索引
areaList.push_back(area); // 面积
*/
if(withPlam)
findPlam(mask,bound[i]);
}
}
/*
int j=usefulList.size()-1;
usefulContoursList.clear();
for(int i=0;j>=0;j--){
i = usefulList.at(j);
float area = areaList.at(j);
if(showRect) {
drawContours(src,points_poly,i,COLOR_GREEN);
}
float arc = arcLength(points_poly[i],true);
// this has some err
if(arc<=0) continue;
cv::Moments m = moments(points[i],false);
if(m.m00<=0) continue;
UsefulContours   my;
my.poly = points_poly[i];
// my.poly = points[i];
my.bound = bound[i];
my.ratio = area/arc;
my.center = cv::Point2f(m.m10/m.m00,m.m01/m.m00);
usefulContoursList.push_back(my);
//findPlam(srcMask,bound[i]);
}
*/
//////////噪声过滤完毕，开始处理图像
skin.setTo(0);
src.copyTo(skin,mask);

#ifdef DEBUG
addWeighted(src,0.5,skin,0.9,1.0,src);
LOGD("findArea(%d*%d): %d us",skin.cols,skin.rows,(int)(getTime(true)-beginT));
#endif
#ifdef  PLAM_INIT_ET
if(!isPlamFound){
if(lastPlamHold.unHoldCount++>30){
lastPlamHold.unHoldCount = 0;
lastPlamHold.used = false;
}
}
#endif
}