[Unity3D——用代码说明一切]Unity结合Kinect2体感开发：Kinect面部识别

实现效果：



Unity版本：5.3.5

项目包下载地址：点击打开链接

关键原理：

1.利用得到的Kinect的Body信息

2.利用Body信息来检测面部的创建和移除，面部追踪ID和Body追踪ID相同

3.新建面部识别源

4.从源中得到面部的各种信息，比如表情状态，旋转，5官点位等等

主要知识点：

1.FaceFrameSource.Create

2.FaceFrameFeatures

3.FaceFrameReader和FaceFrameResult

主要脚本：

KinectFaceControl用于获得Kinect获得的身体信息并根据其创建面部信息

CustomKinectFace用于测试面部各种信息的获取和显示，提供事件监听接口

CamerColorView用于UGUI显示Kinect摄像头内容

using UnityEngine;
using System.Collections;
using Windows.Kinect;
using UnityEngine.UI;

///
/// 摄像机信息 主要原理就是通过Kinect得到图像RGB数据 然后在Texture上绘制
/// 基本逻辑和获取身体信息相同
///
public class CamerColorView : MonoBehaviour
{
//Kinect对象
private KinectSensor _kinectSensor;
//颜色信息读取流
private ColorFrameReader _colorReader;
private Texture2D _textureInfo;
private byte[] _colorData;

private RawImage _cameraMaterial;

// Use this for initialization
void Start ()
{
_cameraMaterial = this.gameObject.GetComponent();
_cameraMaterial.uvRect = new Rect(0, 0, 1, -1);
_kinectSensor = KinectSensor.GetDefault();
if (_kinectSensor == null)
return;

_colorReader = _kinectSensor.ColorFrameSource.OpenReader();

//初始化图片信息
FrameDescription color = _kinectSensor.ColorFrameSource.CreateFrameDescription(ColorImageFormat.Rgba);
_textureInfo = new Texture2D(color.Width, color.Height, TextureFormat.RGBA32, false);
_colorData = new byte[color.BytesPerPixel * color.LengthInPixels];

if (!_kinectSensor.IsOpen)
_kinectSensor.Open();
}

// Update is called once per frame
void Update ()
{
if (_colorReader == null)
return;

ColorFrame frame = _colorReader.AcquireLatestFrame();
if (frame == null)
return;
//存储rgb信息
frame.CopyConvertedFrameDataToArray(_colorData, ColorImageFormat.Rgba);
//texture加载信息
_textureInfo.LoadRawTextureData(_colorData);
//应用
_textureInfo.Apply();

frame.Dispose();
frame = null;

//设置显示信息
_cameraMaterial.texture = _textureInfo;
}
}
using UnityEngine;
using System.Collections;
using Windows.Kinect;
using System.Collections.Generic;
using Microsoft.Kinect.Face;

///
/// 体感游戏对象控制器  主要用于得到检测到的人体信息
///
public class KinectFaceControl : MonoBehaviour
{
//Kinect对象
private KinectSensor _kinectSensor;
//身体信息读取流
private BodyFrameReader _bodyReader;

//从Kinect得到的身体信息 每帧获取
private Body[] _bodyDatas = null;

//场景中的体感对象面部信息
private Dictionary _faces = new Dictionary();
//当前追踪到的游戏对象
private List _nowTrackedIDs = new List();
//用于缓存已知的追踪到的游戏对象
private List _nowCreatedBodyID = new List();

void Start()
{
//得到kinect设备对象
_kinectSensor = KinectSensor.GetDefault();

if (_kinectSensor != null)
{
//得到身体数据流
_bodyReader = _kinectSensor.BodyFrameSource.OpenReader();

//设备是否开启
if (!_kinectSensor.IsOpen)
_kinectSensor.Open();
}
}

void Update()
{
//通过Kinect得到身体信息
GetBodyInfos();
//得到身体信息后 更新场景中的身体对象
UpdateFaseObject();
}

///
/// 得到身体信息
///
private void GetBodyInfos()
{
if (_bodyReader != null)
{
BodyFrame frame = _bodyReader.AcquireLatestFrame();
if (frame != null)
{
//初始化身体数组信息
if (_bodyDatas == null)
_bodyDatas = new Body[_kinectSensor.BodyFrameSource.BodyCount];

//得到该帧的身体信息
frame.GetAndRefreshBodyData(_bodyDatas);

//清楚该帧
frame.Dispose();
frame = null;
}
}
}

///
/// 根据身体信息更新场景上的对象信息   创建  骨骼位置更新等
///
private void UpdateFaseObject()
{
//判断是否得到了身体信息
if (_bodyDatas == null)
return;

_nowTrackedIDs.Clear();
//记录当前追踪到的对象
foreach (Body body in _bodyDatas)
{
if (body == null)
continue;
if (body.IsTracked)
_nowTrackedIDs.Add(body.TrackingId);//trackingid是追踪到对象的唯一ID
}

//记录当前已经有的身体对象
_nowCreatedBodyID = new List(_faces.Keys);
//遍历场景中已经创建的体感游戏对象  看看 是否当前已经不再被追踪了 动态的删除它
foreach (ulong trackingID in _nowCreatedBodyID)
{
//当前kinect已经没有再扫描到对应对象了 那么在场景上删除它 并清楚记录
if (!_nowTrackedIDs.Contains(trackingID))
{
_faces[trackingID].Dispose();
_faces.Remove(trackingID);
}
}

//现在开始更新游戏对象信息 或者动态创建游戏对象
foreach (Body body in _bodyDatas)
{
if (body == null)
continue;
//身体信息是否被追踪
if (body.IsTracked)
{
//场景上没有对应游戏对象 创建对象
if (!_faces.ContainsKey(body.TrackingId))
CreateFaceObject(body.TrackingId);
//更新对象的位置信息
_faces[body.TrackingId].UpdateFace();
}
}
}

///
/// 动态 在场景上创建对象
///
///
private void CreateFaceObject(ulong id)
{
CustomKinectFace info = new CustomKinectFace();
info.CreateKinectFace(_kinectSensor, id);
//监听点信息
info.handleFacePointInfo += HandleFacePointInfo;
//同理 可以在这里监听其它事件  用于处理自己的逻辑

_faces.Add(id, info);
}

///
/// 监听处理脸部点信息
///
///
private void HandleFacePointInfo(Dictionary pointInfoDic)
{
//可以写入需要用点信息处理的逻辑
}

private int index = 0;
///
/// 主要用于显示面部测试信息
///
void OnGUI()
{
index = 0;
foreach (CustomKinectFace faceLogic in _faces.Values)
{
faceLogic.DebugGUIInfo(index);
++index;
}
}
}
using UnityEngine;
using System.Collections;
using Windows.Kinect;
using Microsoft.Kinect.Face;
using System.Collections.Generic;

//处理脸部状态委托
public delegate void HandleFaceProperty(Dictionary property);
//处理脸部角度委托
public delegate void HandleFaceQuaternion(Quaternion quat);
//处理脸部检测框委托
public delegate void HandleFaceRect( RectI rect);
//处理脸部5个点的委托
public delegate void HandleFacePointInfo(Dictionary pointInfoDic);

///
/// 自定义面部类
///
public class CustomKinectFace
{
//面部追踪ID 和身体一样
private ulong _trackingId = 0;
//面部信息流
private FaceFrameReader _faceReader;
//当前帧面部信息结果
private FaceFrameResult _nowResult;
//面部对象 测试时显示面部各个点
private GameObject _faceObject;
//用于画方框 测试时用
private LineRenderer _faceLine;
//是否是第一次创建面部 为了方便观察  把摄像机位置 在鼻子的地方设置一次
private static bool _isFirst = true;
//是否处于测试状态
private static bool _isDebug = true;

///
/// 处理面部点位的事件监听
///
public event HandleFacePointInfo handleFacePointInfo;
///
/// 处理面部追踪范围
///
public event HandleFaceRect handleFaceRect;
///
/// 处理面部旋转角度
///
public event HandleFaceQuaternion handleFaceQuaternion;
///
/// 处理面部状态属性
///
public event HandleFaceProperty handleFaceProperty;

///
/// 创建Kinect面部源信息
///
/// kinect设备
/// 跟踪id
public void CreateKinectFace( KinectSensor sensor, ulong initialTrackingId)
{
_trackingId = initialTrackingId;
//1-从对应设备创建一个面部源信息
//2-这里的跟踪id会爆body检测的id传入 用于标示不同人的面部表情
//3-第三个参数 是你需要获取哪些信息  需要什么 就传入什么 多个 就用 |
FaceFrameSource source = FaceFrameSource.Create(sensor, _trackingId,
FaceFrameFeatures.PointsInColorSpace |
FaceFrameFeatures.BoundingBoxInColorSpace |
FaceFrameFeatures.FaceEngagement |
FaceFrameFeatures.Glasses |
FaceFrameFeatures.Happy |
FaceFrameFeatures.LeftEyeClosed |
FaceFrameFeatures.RightEyeClosed |
FaceFrameFeatures.LookingAway |
FaceFrameFeatures.MouthMoved |
FaceFrameFeatures.MouthOpen|
FaceFrameFeatures.RotationOrientation);

_faceReader = source.OpenReader();
}

///
/// 移除对象
///
public void Dispose()
{
if (_faceObject != null)
GameObject.Destroy(_faceObject);
}

///
/// 更新面部信息
///
public void UpdateFace()
{
if (_faceReader == null)
return;
//得到最后一帧信息
FaceFrame faceFrame = _faceReader.AcquireLatestFrame();
if (faceFrame == null)
return;
if (!faceFrame.IsTrackingIdValid)
return;
//得到面部帧信息中的具体结果信息
_nowResult = faceFrame.FaceFrameResult;
if (_nowResult == null)
return;
//测试状态为true时进入
DebugInfo();

//监听事件处理
HandleEvent();

faceFrame.Dispose();
faceFrame = null;
}

///
/// 处理事件监听
///
public void HandleEvent()
{
if (handleFacePointInfo != null)
handleFacePointInfo(_nowResult.FacePointsInColorSpace);
if (handleFaceProperty != null)
handleFaceProperty(_nowResult.FaceProperties);
if (handleFaceQuaternion != null)
handleFaceQuaternion(GetQuaternionFromeFace(_nowResult.FaceRotationQuaternion));
if (handleFaceRect != null)
handleFaceRect(_nowResult.FaceBoundingBoxInColorSpace);
}

///
/// 得到面部坐标 由于Kinect返回的数值比较大 所以 这里除以100 并且y轴和u3d中是反向的
///
///
///
public Vector3 GetVector3FromFacePoint(Point point)
{
return new Vector3(point.X/100f, -point.Y/100f, 0);
}

///
/// 得到面部角度信息
///
///
///
public Quaternion GetQuaternionFromeFace(Windows.Kinect.Vector4 v4)
{
return new Quaternion(v4.X, v4.Y, v4.Z, v4.W);
}

///
/// 测试信息更新
///
///
private void DebugInfo()
{
//测试状态
if (_isDebug)
{
if (_faceObject == null)
CreateDebugFaceObject();
RefreshDebugFaceObject();
//画方框线
_faceLine.SetPosition(0,new Vector3(_nowResult.FaceBoundingBoxInColorSpace.Left/100f, -_nowResult.FaceBoundingBoxInColorSpace.Top/100f, 0));
_faceLine.SetPosition(1, new Vector3(_nowResult.FaceBoundingBoxInColorSpace.Right / 100f, -_nowResult.FaceBoundingBoxInColorSpace.Top / 100f, 0));
_faceLine.SetPosition(2, new Vector3(_nowResult.FaceBoundingBoxInColorSpace.Right / 100f, -_nowResult.FaceBoundingBoxInColorSpace.Bottom / 100f, 0));
_faceLine.SetPosition(3, new Vector3(_nowResult.FaceBoundingBoxInColorSpace.Left / 100f, -_nowResult.FaceBoundingBoxInColorSpace.Bottom / 100f, 0));
_faceLine.SetPosition(4, new Vector3(_nowResult.FaceBoundingBoxInColorSpace.Left / 100f, -_nowResult.FaceBoundingBoxInColorSpace.Top / 100f, 0));
}
}

///
/// 调试时可用于GUI信息显示  在外部Mono类中的OnGUI中调用 主要用于测试
///
public void DebugGUIInfo(int index)
{
if (_nowResult == null || !_isDebug)
return;
GUI.Label(new Rect(Screen.width - 120*(index+1), 0, 140, 50), "是否开心:" + _nowResult.FaceProperties[FaceProperty.Happy].ToString());
GUI.Label(new Rect(Screen.width - 120*(index + 1), 20, 140, 50), "是否戴眼镜:" + _nowResult.FaceProperties[FaceProperty.WearingGlasses].ToString());
GUI.Label(new Rect(Screen.width - 120 * (index + 1), 40, 140, 50), "左眼是否关闭:" + _nowResult.FaceProperties[FaceProperty.LeftEyeClosed].ToString());
GUI.Label(new Rect(Screen.width - 120 * (index + 1), 60, 140, 50), "右眼是否关闭:" + _nowResult.FaceProperties[FaceProperty.RightEyeClosed].ToString());
GUI.Label(new Rect(Screen.width - 120 * (index + 1), 80, 140, 50), "嘴是否打开:" + _nowResult.FaceProperties[FaceProperty.MouthOpen].ToString());
GUI.Label(new Rect(Screen.width - 120 * (index + 1), 100, 140, 50), "嘴是否移动:" + _nowResult.FaceProperties[FaceProperty.MouthMoved].ToString());
GUI.Label(new Rect(Screen.width - 120 * (index + 1), 120, 140, 50), "是否看四周:" + _nowResult.FaceProperties[FaceProperty.LookingAway].ToString());
Quaternion q = GetQuaternionFromeFace(_nowResult.FaceRotationQuaternion);
GUI.Label(new Rect(Screen.width - 120 * (index + 1), 140, 140, 50), "X轴角度" + q.eulerAngles.x.ToString());
GUI.Label(new Rect(Screen.width - 120 * (index + 1), 160, 140, 50), "y轴角度" + q.eulerAngles.y.ToString());
GUI.Label(new Rect(Screen.width - 120 * (index + 1), 180, 140, 50), "z轴角度" + q.eulerAngles.z.ToString());
}

///
/// 创建面部点
///
///
private void CreateDebugFaceObject()
{
_faceObject = new GameObject("Face:" + _trackingId);
_faceLine = _faceObject.AddComponent();
_faceLine.useWorldSpace = true;
_faceLine.SetVertexCount(5);
_faceLine.SetWidth(0.1f, 0.1f);

foreach (FacePointType type in _nowResult.FacePointsInColorSpace.Keys)
{
GameObject pointObj = GameObject.CreatePrimitive(PrimitiveType.Sphere);

pointObj.transform.localScale = Vector3.one * 0.3f;
pointObj.name = type.ToString();
pointObj.transform.parent = _faceObject.transform;
}
}

///
/// 更新面部对象位置
///
private void RefreshDebugFaceObject()
{
foreach (FacePointType type in _nowResult.FacePointsInColorSpace.Keys)
{
Transform pointObj = _faceObject.transform.Find(type.ToString());
pointObj.transform.localPosition = GetVector3FromFacePoint(_nowResult.FacePointsInColorSpace[type]);

//这里只是第一次 设置射线机的位置 方便观察
if (_isFirst && type == FacePointType.Nose)
{
Camera.main.transform.position = new Vector3(pointObj.transform.position.x, pointObj.transform.position.y, Camera.main.transform.position.z);
_isFirst = false;
}
}
}

}

额外知识点，如何得到Kinect的高清面部信息

关键用法:

//高清面部信息源
HighDefinitionFaceFrameSource source = HighDefinitionFaceFrameSource.Create(sensor);
source.TrackingId = _trackingId;
//信息流
HighDefinitionFaceFrameReader _faceHDReader = source.OpenReader();
//顶点信息
FaceAlignment _faceAlignment = FaceAlignment.Create();
//面部模型点信息
FaceModel _faceModel = FaceModel.Create();
//每一帧的高清面部信息
HighDefinitionFaceFrame faceFrame =  _faceHDReader.AcquireLatestFrame();
//得到顶点和状态
faceFrame.GetAndRefreshFaceAlignmentResult(_faceAlignment);
//得到模型点
_faceModel.CalculateVerticesForAlignment(_faceAlignment)