Realsense（一）提取彩色和深度视频流并实现pxcimage到mat

一、简要介绍

关于realsense的介绍，网上很多，这里不再赘述，sdk及相关文档可参考realsense SDK，也可参考开发人员专区。

运行代码之前，要确保你已经安装好了realsense的DCM和SDK，官网有教程，具体请参考DCM和SDK安装步骤。

二、代码

#include <pxcsensemanager.h>
#include <pxcsession.h>
#include "util_render.h"
#include <iostream>
#include <string>
#include <stdio.h>
#include <opencv2\opencv.hpp>
#include <windows.h>

#define WIDTH 640
#define HEIGHT 480

using namespace cv;
using namespace std;

int main(int argc, char** argv)
{
UtilRender *renderColor = new UtilRender(L"COLOR_STREAM");
UtilRender *renderDepth = new UtilRender(L"DEPTH_STREAM");

PXCSenseManager *psm = 0;
psm = PXCSenseManager::CreateInstance();
if (!psm)
{
wprintf_s(L"Unabel to create the PXCSenseManager\n");
return 1;
}
pxcStatus sts;

psm->EnableStream(PXCCapture::STREAM_TYPE_COLOR, WIDTH, HEIGHT);

psm->EnableStream(PXCCapture::STREAM_TYPE_DEPTH, WIDTH, HEIGHT);

sts = psm->Init();
if (sts != PXC_STATUS_NO_ERROR)
{
wprintf_s(L"Unabel to Initializes the pipeline\n");
return 2;
}

PXCImage *colorIm, *depthIm;
PXCImage::ImageData depth_data,color_data;
PXCImage::ImageInfo depth_info,color_info;
while (psm->AcquireFrame(true) >= PXC_STATUS_NO_ERROR)

{
if (psm->AcquireFrame(true) < PXC_STATUS_NO_ERROR) break;

PXCCapture::Sample *sample = psm->QuerySample();

colorIm = sample->color;
depthIm = sample->depth;

if (colorIm->AcquireAccess(PXCImage::ACCESS_READ, PXCImage::PIXEL_FORMAT_RGB24, &color_data) < PXC_STATUS_NO_ERROR)
wprintf_s(L"未正常获取彩色图\n");
if (depthIm->AcquireAccess(PXCImage::ACCESS_READ, &depth_data) < PXC_STATUS_NO_ERROR)
wprintf_s(L"未正常获取深度图\n");

depth_info = sample->depth->QueryInfo();
color_info = sample->color->QueryInfo();

Mat depth(Size(depth_info.width, depth_info.height), CV_16UC1, (void*)depth_data.planes[0], depth_data.pitches[0] / sizeof(uchar));
Mat color(Size(color_info.width, color_info.height), CV_8UC3, (void*)color_data.planes[0], color_data.pitches[0] / sizeof(uchar));

depthIm->ReleaseAccess(&depth_data);
colorIm->ReleaseAccess(&color_data);

if (!renderColor->RenderFrame(colorIm)) break;
if (!renderDepth->RenderFrame(depthIm)) break;

psm->ReleaseFrame();

imshow("color", color);
waitKey(1);
//CV_16UC1的图片在imshow时会除以256，将最远探测距离设为z，那么imshow时可以乘以255*256/z,此处乘以15
imshow("depth", depth * 15);
waitKey(1);

}
psm->Release();
system("pause");
}

三、相关解释

新建项目，写入源代码，生成解决方案之前，记得要在项目的属性管理器右键项目，选择添加现有属性表，选择C:\Program Files (x86)\Intel\RSSDK\props目录下的一个官方配置好的属性表（前提是DCM和SDK的安装路径是默认路径，没有自行修改）；另外因为我们还要进行cv::Mat类型的转化，所以也要导入opencv的属性表。

运行之后，可以看到4个窗口，COLOR_STREAM和DEPTH_STREAM为分别realsense提取到的彩色视频流和深度视频流；color和depth分别为我们转化为opencv中mat类型之后的彩色和深度视频流。



目前实现pxcimage到mat类型转化的方法有好几种，大体类似，主要是对PXCImage::ImageData中的plane和pitch的理解，博主理解为plane[0]为该sdk中图片数据的首地址，pitches可参考http://www.cnblogs.com/gamedes/p/4541765.html。

也看到论坛上有人专门写过pxcimage到mat类型的转化函数，函数代码如下：

void ConvertPXCImageToOpenCVMat(PXCImage *inImg, Mat *outImg) {
int cvDataType;
int cvDataWidth;

PXCImage::ImageData data;
inImg->AcquireAccess(PXCImage::ACCESS_READ, &data);
PXCImage::ImageInfo imgInfo = inImg->QueryInfo();

switch (data.format) {
/* STREAM_TYPE_COLOR */
case PXCImage::PIXEL_FORMAT_YUY2: /* YUY2 image  */
case PXCImage::PIXEL_FORMAT_NV12: /* NV12 image */
throw(0); // Not implemented
case PXCImage::PIXEL_FORMAT_RGB32: /* BGRA layout on a little-endian machine */
cvDataType = CV_8UC4;
cvDataWidth = 4;
break;
case PXCImage::PIXEL_FORMAT_RGB24: /* BGR layout on a little-endian machine */
cvDataType = CV_8UC3;
cvDataWidth = 3;
break;
case PXCImage::PIXEL_FORMAT_Y8:  /* 8-Bit Gray Image, or IR 8-bit */
cvDataType = CV_8U;
cvDataWidth = 1;
break;

/* STREAM_TYPE_DEPTH */
case PXCImage::PIXEL_FORMAT_DEPTH: /* 16-bit unsigned integer with precision mm. */
case PXCImage::PIXEL_FORMAT_DEPTH_RAW: /* 16-bit unsigned integer with device specific precision (call device->QueryDepthUnit()) */
cvDataType = CV_16U;
cvDataWidth = 2;
break;
case PXCImage::PIXEL_FORMAT_DEPTH_F32: /* 32-bit float-point with precision mm. */
cvDataType = CV_32F;
cvDataWidth = 4;
break;

/* STREAM_TYPE_IR */
case PXCImage::PIXEL_FORMAT_Y16:          /* 16-Bit Gray Image */
cvDataType = CV_16U;
cvDataWidth = 2;
break;
case PXCImage::PIXEL_FORMAT_Y8_IR_RELATIVE:    /* Relative IR Image */
cvDataType = CV_8U;
cvDataWidth = 1;
break;
}

// suppose that no other planes
if (data.planes[1] != NULL) throw(0); // not implemented
// suppose that no sub pixel padding needed
if (data.pitches[0] % cvDataWidth!=0) throw(0); // not implemented

outImg->create(imgInfo.height, data.pitches[0] / cvDataWidth, cvDataType);

memcpy(outImg->data, data.planes[0], imgInfo.height*imgInfo.width*cvDataWidth*sizeof(pxcBYTE));

inImg->ReleaseAccess(&data);
}

原则上传入pxcimage的指针，再通过case根据原图像类型选择要转化的图像类型是没有问题的，博主尝试过使用这个函数，但总会出现R1060的错误，似乎是指针非法使用。大家可以自行尝试一下，成功的小伙伴麻烦告知一下博主，让博主学习学习。