[Unity3D——用代码说明一切]Unity结合Kinect2体感开发：Kinect控制U3D中的模型

实现效果：



Kinect1和Kinect2的关键骨骼位置信息：



Unity版本：5.3.5

项目包下载地址：点击打开链接

关键原理：

1.利用得到的Kinect的Body信息得到所有骨骼信息

2.通过骨骼信息得到对应点骨骼的旋转信息

3.将旋转信息用到本地的模型骨骼上

4.每个骨骼点的方向向量的计算

主要知识点：

要理解坐标系中Vector3和Quaternion之间的关系,世界坐标和相对坐标的相互转换

主要脚本：

KinectBodyControl用于获得Kinect获得的身体信息并创建模型

KinectBodyControlLogic用于控制模型骨骼的更新

CamerColorView用于UGUI显示Kinect摄像头内容

using UnityEngine;
using System.Collections;
using Windows.Kinect;
using System.Collections.Generic;

///
/// 体感游戏对象控制器
///
public class KinectBodyControl : MonoBehaviour
{
//Kinect对象
private KinectSensor _kinectSensor;
//身体信息读取流
private BodyFrameReader _bodyReader;

//从Kinect得到的身体信息 每帧获取
private Body[] _bodyDatas = null;

//场景中的体感游戏对象
private Dictionary _bodies = new Dictionary();
//当前追踪到的游戏对象
private List _nowTrackedIDs = new List();
//用于缓存已知的追踪到的游戏对象
private List _nowCreatedBodyID = new List();

void Start()
{
//得到kinect设备对象
_kinectSensor = KinectSensor.GetDefault();

if (_kinectSensor != null)
{
//得到身体数据流
_bodyReader = _kinectSensor.BodyFrameSource.OpenReader();

//设备是否开启
if (!_kinectSensor.IsOpen)
_kinectSensor.Open();
}
}

void Update()
{
//通过Kinect得到身体信息
GetBodyInfos();
//得到身体信息后 更新场景中的身体对象
UpdateBodyObject();
}

///
/// 得到身体信息
///
private void GetBodyInfos()
{
if (_bodyReader != null)
{
BodyFrame frame = _bodyReader.AcquireLatestFrame();
if (frame != null)
{
//初始化身体数组信息
if (_bodyDatas == null)
_bodyDatas = new Body[_kinectSensor.BodyFrameSource.BodyCount];

//得到该帧的身体信息
frame.GetAndRefreshBodyData(_bodyDatas);

//清楚该帧
frame.Dispose();
frame = null;
}
}
}

///
/// 根据身体信息更新场景上的对象信息   创建  骨骼位置更新等
///
private void UpdateBodyObject()
{
//判断是否得到了身体信息
if (_bodyDatas == null)
return;

_nowTrackedIDs.Clear();
//记录当前追踪到的对象
foreach (Body body in _bodyDatas)
{
if (body == null)
continue;
if (body.IsTracked)
_nowTrackedIDs.Add(body.TrackingId);//trackingid是追踪到对象的唯一ID
}

//记录当前已经有的身体对象
_nowCreatedBodyID = new List(_bodies.Keys);
//遍历场景中已经创建的体感游戏对象  看看 是否当前已经不再被追踪了 动态的删除它
foreach (ulong trackingID in _nowCreatedBodyID)
{
//当前kinect已经没有再扫描到对应对象了 那么在场景上删除它 并清楚记录
if (!_nowTrackedIDs.Contains(trackingID))
{
Destroy(_bodies[trackingID].gameObject);
_bodies.Remove(trackingID);
}
}

//现在开始更新游戏对象信息 或者动态创建游戏对象
foreach (Body body in _bodyDatas)
{
if (body == null)
continue;
//身体信息是否被追踪
if (body.IsTracked)
{
//场景上没有对应游戏对象 创建对象
if (!_bodies.ContainsKey(body.TrackingId))
CreateBodyObject(body.TrackingId);
//更新对象的位置信息
_bodies[body.TrackingId].RefreshBodyInfo(body);
}
}
}

///
/// 动态 在场景上创建对象
///
///
private void CreateBodyObject(ulong id)
{
GameObject obj = Instantiate(Resources.Load("RoleObject/GUJIA")) as GameObject;
obj.name = "Body:" + id;
KinectBodyControlLogic info = obj.GetComponent();
_bodies.Add(id, info);
}
}
using UnityEngine;
using System.Collections;
using Windows.Kinect;
using System.Collections.Generic;

///
/// 身体控制逻辑
///
public class KinectBodyControlLogic : MonoBehaviour
{
///
/// 根骨骼
///
public GameObject SpineBase;
///
/// 腰部
///
public GameObject SpineMid;
///
/// 脖子
///
public GameObject Neck;
///
/// 头
///
public GameObject Head;
///
/// 左侧肩膀
///
public GameObject ShoulderLeft;
///
/// 左侧手肘
///
public GameObject ElbowLeft;
///
/// 左侧手腕
///
public GameObject WristLeft;
///
/// 左侧手掌
///
public GameObject HandLeft;
///
/// 右侧肩膀
///
public GameObject ShoulderRight;
///
/// 右侧手肘
///
public GameObject ElbowRight;
///
/// 右侧手腕
///
public GameObject WristRight;
///
/// 右侧手掌
///
public GameObject HandRight;
///
/// 左侧胯部
///
public GameObject HipLeft;
///
/// 左侧膝盖
///
public GameObject KneeLeft;
///
/// 左侧脚腕
///
public GameObject AnkleLeft;
///
/// 左侧脚掌
///
public GameObject FootLeft;
///
/// 右侧胯部
///
public GameObject HipRight;
///
/// 右侧膝盖
///
public GameObject KneeRight;
///
/// 右侧脚踝
///
public GameObject AnkleRight;
///
/// 右侧脚掌
///
public GameObject FootRight;
///
/// 肩部中央位置
///
public GameObject SpineShoulder;
///
/// 左侧指尖
///
public GameObject HandTipLeft;
///
/// 左侧拇指尖
///
public GameObject ThumbLeft;
///
/// 右侧指尖
///
public GameObject HandTipRight;
///
/// 右侧  侧拇指尖
///
public GameObject ThumbRight;

///
/// 是否是用Kinect镜像移动 为true 就像照镜子  为false就像背面控制模型
///
public bool IsMirror = true;

//方向向量配对
///
/// 根骨骼
///
public JointType SpineBaseDir = JointType.SpineMid;
///
/// 腰部
///
public JointType SpineMidDir = JointType.SpineShoulder;
///
/// 脖子
///
public JointType NeckDir = JointType.Head;
///
/// 头
///
public JointType HeadDir = JointType.Head;
///
/// 左侧肩膀
///
public JointType ShoulderLeftDir = JointType.ElbowLeft;
///
/// 左侧手肘
///
public JointType ElbowLeftDir = JointType.WristLeft;
///
/// 左侧手腕
///
public JointType WristLeftDir = JointType.HandLeft;
///
/// 左侧手掌
///
public JointType HandLeftDir = JointType.HandTipLeft;
///
/// 右侧肩膀
///
public JointType ShoulderRightDir = JointType.ElbowRight;
///
/// 右侧手肘
///
public JointType ElbowRightDir = JointType.WristRight;
///
/// 右侧手腕
///
public JointType WristRightDir = JointType.HandRight;
///
/// 右侧手掌
///
public JointType HandRightDir = JointType.HandTipRight;
///
/// 左侧胯部
///
public JointType HipLeftDir = JointType.KneeLeft;
///
/// 左侧膝盖
///
public JointType KneeLeftDir = JointType.AnkleLeft;
///
/// 左侧脚腕
///
public JointType AnkleLeftDir = JointType.FootLeft;
///
/// 左侧脚掌
///
public JointType FootLeftDir = JointType.FootLeft;
///
/// 右侧胯部
///
public JointType HipRightDir = JointType.KneeRight;
///
/// 右侧膝盖
///
public JointType KneeRightDir = JointType.AnkleRight;
///
/// 右侧脚踝
///
public JointType AnkleRightDir = JointType.FootRight;
///
/// 右侧脚掌
///
public JointType FootRightDir = JointType.FootRight;
///
/// 肩部中央位置
///
public JointType SpineShoulderDir = JointType.Neck;
///
/// 左侧指尖
///
public JointType HandTipLeftDir = JointType.HandTipLeft;
///
/// 左侧拇指尖
///
public JointType ThumbLeftDir = JointType.ThumbLeft;
///
/// 右侧指尖
///
public JointType HandTipRightDir = JointType.HandTipRight;
///
/// 右侧  侧拇指尖
///
public JointType ThumbRightDir = JointType.ThumbRight;

///
/// 骨骼信息字典 对应Kinect的骨骼类型
///
private Dictionary _boneObjects;
///
/// 骨骼方向向量计算表示 用于骨骼方向向量计算
///
private Dictionary _boneDirTypes;

//模型骨骼默认方向向量
private Dictionary _bonesDir = new Dictionary();
//模型骨骼默认旋转四元数信息
private Dictionary _bonesRotation = new Dictionary();

//根骨骼的方向向量  之后用于判断 人物朝向
private Vector3 _rootDir;
//上半身腰部骨骼的方向向量  用于之后判断 上半身朝向
private Vector3 _upperBodyDir;

//是否已经初始化
private bool _isInit = false;

void Awake()
{
//初始化骨骼信息
InitBoneInfo();
//初始化方向
InitVector();
}

///
/// 初始化骨骼字典信息
///
private void InitBoneInfo()
{
//初始化骨骼对象字典
if(IsMirror)
{
_boneObjects = new Dictionary()
{
{ JointType.SpineBase, SpineBase},
{ JointType.SpineMid, SpineMid},
{ JointType.Neck, Neck},
{ JointType.Head, Head},
{ JointType.ShoulderLeft, ShoulderRight},
{ JointType.ElbowLeft, ElbowRight},
{ JointType.WristLeft, WristRight},
{ JointType.HandLeft, HandRight},
{ JointType.ShoulderRight, ShoulderLeft},
{ JointType.ElbowRight, ElbowLeft},
{ JointType.WristRight, WristLeft},
{ JointType.HandRight, HandLeft},
{ JointType.HipLeft, HipRight},
{ JointType.KneeLeft, KneeRight},
{ JointType.AnkleLeft, AnkleRight},
{ JointType.FootLeft, FootRight},
{ JointType.HipRight, HipLeft},
{ JointType.KneeRight, KneeLeft},
{ JointType.AnkleRight, AnkleLeft},
{ JointType.FootRight, FootLeft},
{ JointType.SpineShoulder, SpineShoulder},
{ JointType.HandTipLeft, HandTipRight},
{ JointType.ThumbLeft, ThumbRight},
{ JointType.HandTipRight, HandTipLeft},
{ JointType.ThumbRight, ThumbLeft},
};
}
else
{
_boneObjects = new Dictionary()
{
{ JointType.SpineBase, SpineBase},
{ JointType.SpineMid, SpineMid},
{ JointType.Neck, Neck},
{ JointType.Head, Head},
{ JointType.ShoulderLeft, ShoulderLeft},
{ JointType.ElbowLeft, ElbowLeft},
{ JointType.WristLeft, WristLeft},
{ JointType.HandLeft, HandLeft},
{ JointType.ShoulderRight, ShoulderRight},
{ JointType.ElbowRight, ElbowRight},
{ JointType.WristRight, WristRight},
{ JointType.HandRight, HandRight},
{ JointType.HipLeft, HipLeft},
{ JointType.KneeLeft, KneeLeft},
{ JointType.AnkleLeft, AnkleLeft},
{ JointType.FootLeft, FootLeft},
{ JointType.HipRight, HipRight},
{ JointType.KneeRight, KneeRight},
{ JointType.AnkleRight, AnkleRight},
{ JointType.FootRight, FootRight},
{ JointType.SpineShoulder, SpineShoulder},
{ JointType.HandTipLeft, HandTipLeft},
{ JointType.ThumbLeft, ThumbLeft},
{ JointType.HandTipRight, HandTipRight},
{ JointType.ThumbRight, ThumbRight},
};
}

//初始化 各骨骼的默认方向向量
_boneDirTypes = new Dictionary()
{
{ JointType.SpineBase, SpineBaseDir},
{ JointType.SpineMid, SpineMidDir},
{ JointType.Neck, NeckDir},
{ JointType.Head, HeadDir},
{ JointType.ShoulderLeft, ShoulderLeftDir},
{ JointType.ElbowLeft, ElbowLeftDir},
{ JointType.WristLeft, WristLeftDir},
{ JointType.HandLeft, HandLeftDir},
{ JointType.ShoulderRight, ShoulderRightDir},
{ JointType.ElbowRight, ElbowRightDir},
{ JointType.WristRight, WristRightDir},
{ JointType.HandRight, HandRightDir},
{ JointType.HipLeft, HipLeftDir},
{ JointType.KneeLeft, KneeLeftDir},
{ JointType.AnkleLeft, AnkleLeftDir},
{ JointType.FootLeft, FootLeftDir},
{ JointType.HipRight, HipRightDir},
{ JointType.KneeRight, KneeRightDir},
{ JointType.AnkleRight, AnkleRightDir},
{ JointType.FootRight, FootRightDir},
{ JointType.SpineShoulder, SpineShoulderDir},
{ JointType.HandTipLeft, HandTipLeftDir},
{ JointType.ThumbLeft, ThumbLeftDir},
{ JointType.HandTipRight, HandTipRightDir},
{ JointType.ThumbRight, ThumbRightDir},
};
}

///
/// 初始化方向向量等信息
///
private void InitVector()
{
//这六个骨骼用于控制模型朝向  和 上半身朝向
if( _boneObjects[JointType.HipRight] == null ||
_boneObjects[JointType.HipLeft] == null ||
_boneObjects[JointType.SpineBase] == null ||
_boneObjects[JointType.ShoulderRight] == null ||
_boneObjects[JointType.ShoulderLeft] == null ||
_boneObjects[JointType.SpineMid] == null)
{
Debug.LogError("关键骨骼为空");
}

//根骨骼的方向向量 为 右侧胯骨减去左侧胯骨
_rootDir = _boneObjects[JointType.HipRight].transform.position - _boneObjects[JointType.HipLeft].transform.position;
//转换为相对于根节点的相对坐标
_rootDir = _boneObjects[JointType.SpineBase].transform.InverseTransformDirection(_rootDir);

//上半身骨骼的方向向量 为 右侧肩部减去左侧肩部
_upperBodyDir = _boneObjects[JointType.ShoulderRight].transform.position - _boneObjects[JointType.ShoulderLeft].transform.position;
//转换为相对于腰部点的相对坐标
_upperBodyDir = _boneObjects[JointType.SpineMid].transform.InverseTransformDirection(_upperBodyDir);

GameObject boneObj = null;
foreach( JointType type in _boneObjects.Keys )
{
boneObj = _boneObjects[type];
if (boneObj == null)
continue;
//初始旋转信息
_bonesRotation.Add( type, boneObj.transform.localRotation );
//初始方向向量信息
_bonesDir.Add(type, _boneObjects[_boneDirTypes[type]].transform.position - _boneObjects[type].transform.position);
//世界坐标转换为局部坐标
_bonesDir[type] = boneObj.transform.InverseTransformDirection(_bonesDir[type]);
}
_isInit = true;
}

///
/// 跟新身体信息
///
///
public void RefreshBodyInfo(Body body)
{
if (!_isInit)
return;
//Kinect得到的Body骨骼信息
Windows.Kinect.Joint sourceJoint;
GameObject boneObj;
Vector3 boneDir;
Vector3 targetDir;
//更新模型的所有骨骼信息
foreach ( JointType jt in _boneObjects.Keys )
{
boneObj = _boneObjects[jt];
if (boneObj == null)
continue;
//得到Kinect的body信息中对应骨骼点信息
sourceJoint = body.Joints[jt];
//初始化角度
boneObj.transform.localRotation = _bonesRotation[jt];

//初始的方向向量
boneDir = _bonesDir[jt];
//当前kinect中对应骨骼的方向向量
targetDir = GetVector3FromJoint(body.Joints[_boneDirTypes[jt]]) - GetVector3FromJoint(sourceJoint);
//Kinect坐标转U3D世界坐标
targetDir = transform.TransformDirection(targetDir);
//从U3D世界坐标转相对坐标
targetDir = boneObj.transform.InverseTransformDirection(targetDir);
//得到相对角度变换
Quaternion nowQuat = Quaternion.FromToRotation(boneDir, targetDir);

//根骨骼的方向变化
if(jt == JointType.SpineBase)
{
//之前得到的胯骨平行方向向量
boneDir = _rootDir;
//得到当前kinect的肩部骨骼方向向量位置
targetDir = GetVector3FromJoint(body.Joints[JointType.HipRight]) - GetVector3FromJoint(body.Joints[JointType.HipLeft]);
//坐标转换 因为不希望在x轴上旋转 让角色平稳的站在地上 所以 让其在x轴上取投影
targetDir = Vector3.Project(targetDir, transform.right);
targetDir = transform.TransformDirection(targetDir);
targetDir = boneObj.transform.InverseTransformDirection(targetDir);

nowQuat *= Quaternion.FromToRotation(boneDir, targetDir);
}
//上半身骨骼的方向变化
else if (jt == JointType.SpineMid)
{
//上半身平行位置
boneDir = _upperBodyDir;
//得到当前kinect的肩部骨骼方向向量位置
targetDir = GetVector3FromJoint(body.Joints[JointType.ShoulderRight]) - GetVector3FromJoint(body.Joints[JointType.ShoulderLeft]);
//坐标转换
targetDir = transform.TransformDirection(targetDir);
targetDir = boneObj.transform.InverseTransformDirection(targetDir);

nowQuat *= Quaternion.FromToRotation(boneDir, targetDir);
}

//开始转
nowQuat = Quaternion.Lerp(Quaternion.identity, nowQuat, 1);
boneObj.transform.localRotation = boneObj.transform.localRotation * nowQuat;

//更新人物位置
if (jt == JointType.SpineBase)
this.transform.position = GetVector3FromJoint(sourceJoint);
}
}

///
/// 把Kinect中的骨骼关节点 转换为U3D中的Vector3
///
///
///
private Vector3 GetVector3FromJoint(Windows.Kinect.Joint joint)
{
return new Vector3(joint.Position.X, joint.Position.Y, IsMirror ? joint.Position.Z : -joint.Position.Z);
}
}
using UnityEngine;
using System.Collections;
using Windows.Kinect;
using UnityEngine.UI;

///
/// 摄像机信息 主要原理就是通过Kinect得到图像RGB数据 然后在Texture上绘制
/// 基本逻辑和获取身体信息相同
///
public class CamerColorView : MonoBehaviour
{
//Kinect对象
private KinectSensor _kinectSensor;
//颜色信息读取流
private ColorFrameReader _colorReader;
private Texture2D _textureInfo;
private byte[] _colorData;

private RawImage _cameraMaterial;

// Use this for initialization
void Start ()
{
_cameraMaterial = this.gameObject.GetComponent();
_cameraMaterial.uvRect = new Rect(0, 0, 1, -1);
_kinectSensor = KinectSensor.GetDefault();
if (_kinectSensor == null)
return;

_colorReader = _kinectSensor.ColorFrameSource.OpenReader();

//初始化图片信息
FrameDescription color = _kinectSensor.ColorFrameSource.CreateFrameDescription(ColorImageFormat.Rgba);
_textureInfo = new Texture2D(color.Width, color.Height, TextureFormat.RGBA32, false);
_colorData = new byte[color.BytesPerPixel * color.LengthInPixels];

if (!_kinectSensor.IsOpen)
_kinectSensor.Open();
}

// Update is called once per frame
void Update ()
{
if (_colorReader == null)
return;

ColorFrame frame = _colorReader.AcquireLatestFrame();
if (frame == null)
return;
//存储rgb信息
frame.CopyConvertedFrameDataToArray(_colorData, ColorImageFormat.Rgba);
//texture加载信息
_textureInfo.LoadRawTextureData(_colorData);
//应用
_textureInfo.Apply();

frame.Dispose();
frame = null;

//设置显示信息
_cameraMaterial.texture = _textureInfo;
}
}