live555实现ffmpeg解码H264的rtsp流

由于需要实现一个解码H264的rtsp流的web客户端。我首先想到的是live555+ffmpeg。live555用于接收rtsp流，ffmpeg用于解码H264用于显示。看了一下live555发现里面的例子里只有一个openrtsp的例子有点想象，但是那个只是接收rtsp流存在一个文件中。我先尝试写了一个ffmpeg解码H264文件的程序，调试通过。现在只要把live555的例子改一下就可以了，把两个程序联合起来就可以了。这里主要的关键点是找到openrtsp写入文件的地方，只需将这个地方的数据获取到解码显示就可以了。

由于项目忙，也只能抽出时间来记录一下。

main函数在playCommon.cpp。main()的流程比较简单，跟服务端差别不大：建立任务计划对象－－建立环境对象－－处理用户输入的参数(RTSP地址)－－创建RTSPClient实例－－发出第一个RTSP请求（可能是OPTIONS也可能是DESCRIBE）－－进入Loop。

我们主要来看看创建RTPSource在函数createSourceObjects()中，看一下：

[cpp] view
plaincopy

Boolean MediaSubsession::createSourceObjects(int useSpecialRTPoffset) {

do {

// First, check "fProtocolName"

if (strcmp(fProtocolName, "UDP") == 0) {

// A UDP-packetized stream (*not* a RTP stream)

fReadSource = BasicUDPSource::createNew(env(), fRTPSocket);

fRTPSource = NULL; // Note!

if (strcmp(fCodecName, "MP2T") == 0) { // MPEG-2 Transport Stream

fReadSource = MPEG2TransportStreamFramer::createNew(env(), fReadSource);

// this sets "durationInMicroseconds" correctly, based on the PCR values

}

} else {

// Check "fCodecName" against the set of codecs that we support,

// and create our RTP source accordingly

// (Later make this code more efficient, as this set grows #####)

// (Also, add more fmts that can be implemented by SimpleRTPSource#####)

Boolean createSimpleRTPSource = False; // by default; can be changed below

Boolean doNormalMBitRule = False; // default behavior if "createSimpleRTPSource" is True

if (strcmp(fCodecName, "QCELP") == 0) { // QCELP audio

fReadSource =

QCELPAudioRTPSource::createNew(env(), fRTPSocket, fRTPSource,

fRTPPayloadFormat,

fRTPTimestampFrequency);

// Note that fReadSource will differ from fRTPSource in this case

} else if (strcmp(fCodecName, "AMR") == 0) { // AMR audio (narrowband)

fReadSource =

AMRAudioRTPSource::createNew(env(), fRTPSocket, fRTPSource,

fRTPPayloadFormat, 0 /*isWideband*/,

fNumChannels, fOctetalign, fInterleaving,

fRobustsorting, fCRC);

// Note that fReadSource will differ from fRTPSource in this case

} else if (strcmp(fCodecName, "AMR-WB") == 0) { // AMR audio (wideband)

fReadSource =

AMRAudioRTPSource::createNew(env(), fRTPSocket, fRTPSource,

fRTPPayloadFormat, 1 /*isWideband*/,

fNumChannels, fOctetalign, fInterleaving,

fRobustsorting, fCRC);

// Note that fReadSource will differ from fRTPSource in this case

} else if (strcmp(fCodecName, "MPA") == 0) { // MPEG-1 or 2 audio

fReadSource = fRTPSource

= MPEG1or2AudioRTPSource::createNew(env(), fRTPSocket,

fRTPPayloadFormat,

fRTPTimestampFrequency);

} else if (strcmp(fCodecName, "MPA-ROBUST") == 0) { // robust MP3 audio

fReadSource = fRTPSource

= MP3ADURTPSource::createNew(env(), fRTPSocket, fRTPPayloadFormat,

fRTPTimestampFrequency);

if (fRTPSource == NULL) break;

if (!fReceiveRawMP3ADUs) {

// Add a filter that deinterleaves the ADUs after depacketizing them:

MP3ADUdeinterleaver* deinterleaver

= MP3ADUdeinterleaver::createNew(env(), fRTPSource);

if (deinterleaver == NULL) break;

// Add another filter that converts these ADUs to MP3 frames:

fReadSource = MP3FromADUSource::createNew(env(), deinterleaver);

}

} else if (strcmp(fCodecName, "X-MP3-DRAFT-00") == 0) {

// a non-standard variant of "MPA-ROBUST" used by RealNetworks

// (one 'ADU'ized MP3 frame per packet; no headers)

fRTPSource

= SimpleRTPSource::createNew(env(), fRTPSocket, fRTPPayloadFormat,

fRTPTimestampFrequency,

"audio/MPA-ROBUST" /*hack*/);

if (fRTPSource == NULL) break;

// Add a filter that converts these ADUs to MP3 frames:

fReadSource = MP3FromADUSource::createNew(env(), fRTPSource,

False /*no ADU header*/);

} else if (strcmp(fCodecName, "MP4A-LATM") == 0) { // MPEG-4 LATM audio

fReadSource = fRTPSource

= MPEG4LATMAudioRTPSource::createNew(env(), fRTPSocket,

fRTPPayloadFormat,

fRTPTimestampFrequency);

} else if (strcmp(fCodecName, "VORBIS") == 0) { // Vorbis audio

fReadSource = fRTPSource

= VorbisAudioRTPSource::createNew(env(), fRTPSocket,

fRTPPayloadFormat,

fRTPTimestampFrequency);

} else if (strcmp(fCodecName, "VP8") == 0) { // VP8 video

fReadSource = fRTPSource

= VP8VideoRTPSource::createNew(env(), fRTPSocket,

fRTPPayloadFormat,

fRTPTimestampFrequency);

} else if (strcmp(fCodecName, "AC3") == 0 || strcmp(fCodecName, "EAC3") == 0) { // AC3 audio

fReadSource = fRTPSource

= AC3AudioRTPSource::createNew(env(), fRTPSocket,

fRTPPayloadFormat,

fRTPTimestampFrequency);

} else if (strcmp(fCodecName, "MP4V-ES") == 0) { // MPEG-4 Elementary Stream video

fReadSource = fRTPSource

= MPEG4ESVideoRTPSource::createNew(env(), fRTPSocket,

fRTPPayloadFormat,

fRTPTimestampFrequency);

} else if (strcmp(fCodecName, "MPEG4-GENERIC") == 0) {

fReadSource = fRTPSource

= MPEG4GenericRTPSource::createNew(env(), fRTPSocket,

fRTPPayloadFormat,

fRTPTimestampFrequency,

fMediumName, fMode,

fSizelength, fIndexlength,

fIndexdeltalength);

} else if (strcmp(fCodecName, "MPV") == 0) { // MPEG-1 or 2 video

fReadSource = fRTPSource

= MPEG1or2VideoRTPSource::createNew(env(), fRTPSocket,

fRTPPayloadFormat,

fRTPTimestampFrequency);

} else if (strcmp(fCodecName, "MP2T") == 0) { // MPEG-2 Transport Stream

fRTPSource = SimpleRTPSource::createNew(env(), fRTPSocket, fRTPPayloadFormat,

fRTPTimestampFrequency, "video/MP2T",

0, False);

fReadSource = MPEG2TransportStreamFramer::createNew(env(), fRTPSource);

// this sets "durationInMicroseconds" correctly, based on the PCR values

} else if (strcmp(fCodecName, "H261") == 0) { // H.261

fReadSource = fRTPSource

= H261VideoRTPSource::createNew(env(), fRTPSocket,

fRTPPayloadFormat,

fRTPTimestampFrequency);

} else if (strcmp(fCodecName, "H263-1998") == 0 ||

strcmp(fCodecName, "H263-2000") == 0) { // H.263+

fReadSource = fRTPSource

= H263plusVideoRTPSource::createNew(env(), fRTPSocket,

fRTPPayloadFormat,

fRTPTimestampFrequency);

} else if (strcmp(fCodecName, "H264") == 0) {

fReadSource = fRTPSource

= H264VideoRTPSource::createNew(env(), fRTPSocket,

fRTPPayloadFormat,

fRTPTimestampFrequency);

} else if (strcmp(fCodecName, "DV") == 0) {

fReadSource = fRTPSource

= DVVideoRTPSource::createNew(env(), fRTPSocket,

fRTPPayloadFormat,

fRTPTimestampFrequency);

} else if (strcmp(fCodecName, "JPEG") == 0) { // motion JPEG

fReadSource = fRTPSource

= JPEGVideoRTPSource::createNew(env(), fRTPSocket,

fRTPPayloadFormat,

fRTPTimestampFrequency,

videoWidth(),

videoHeight());

} else if (strcmp(fCodecName, "X-QT") == 0

|| strcmp(fCodecName, "X-QUICKTIME") == 0) {

// Generic QuickTime streams, as defined in

// <http://developer.apple.com/quicktime/icefloe/dispatch026.html>

char* mimeType

= new char[strlen(mediumName()) + strlen(codecName()) + 2] ;

sprintf(mimeType, "%s/%s", mediumName(), codecName());

fReadSource = fRTPSource

= QuickTimeGenericRTPSource::createNew(env(), fRTPSocket,

fRTPPayloadFormat,

fRTPTimestampFrequency,

mimeType);

delete[] mimeType;

} else if ( strcmp(fCodecName, "PCMU") == 0 // PCM u-law audio

|| strcmp(fCodecName, "GSM") == 0 // GSM audio

|| strcmp(fCodecName, "DVI4") == 0 // DVI4 (IMA ADPCM) audio

|| strcmp(fCodecName, "PCMA") == 0 // PCM a-law audio

|| strcmp(fCodecName, "MP1S") == 0 // MPEG-1 System Stream

|| strcmp(fCodecName, "MP2P") == 0 // MPEG-2 Program Stream

|| strcmp(fCodecName, "L8") == 0 // 8-bit linear audio

|| strcmp(fCodecName, "L16") == 0 // 16-bit linear audio

|| strcmp(fCodecName, "L20") == 0 // 20-bit linear audio (RFC 3190)

|| strcmp(fCodecName, "L24") == 0 // 24-bit linear audio (RFC 3190)

|| strcmp(fCodecName, "G726-16") == 0 // G.726, 16 kbps

|| strcmp(fCodecName, "G726-24") == 0 // G.726, 24 kbps

|| strcmp(fCodecName, "G726-32") == 0 // G.726, 32 kbps

|| strcmp(fCodecName, "G726-40") == 0 // G.726, 40 kbps

|| strcmp(fCodecName, "SPEEX") == 0 // SPEEX audio

|| strcmp(fCodecName, "T140") == 0 // T.140 text (RFC 4103)

|| strcmp(fCodecName, "DAT12") == 0 // 12-bit nonlinear audio (RFC 3190)

) {

createSimpleRTPSource = True;

useSpecialRTPoffset = 0;

} else if (useSpecialRTPoffset >= 0) {

// We don't know this RTP payload format, but try to receive

// it using a 'SimpleRTPSource' with the specified header offset:

createSimpleRTPSource = True;

} else {

env().setResultMsg("RTP payload format unknown or not supported");

break;

}

if (createSimpleRTPSource) {

char* mimeType

= new char[strlen(mediumName()) + strlen(codecName()) + 2] ;

sprintf(mimeType, "%s/%s", mediumName(), codecName());

fReadSource = fRTPSource

= SimpleRTPSource::createNew(env(), fRTPSocket, fRTPPayloadFormat,

fRTPTimestampFrequency, mimeType,

(unsigned)useSpecialRTPoffset,

doNormalMBitRule);

delete[] mimeType;

}

}

return True;

} while (0);

return False; // an error occurred

}

可以看到这里对于h264是

[cpp] view
plaincopy

fReadSource = fRTPSource

= H264VideoRTPSource::createNew(env(), fRTPSocket,

fRTPPayloadFormat,

fRTPTimestampFrequency);

在MediaSubsession中把fReadSource和fRTPSource初始化了。

socket建立了，Source也创建了，下一步应该是连接Sink，形成一个流。到此为止还未看到Sink的影子，应该是在下一步SETUP中建立，我们看到在continueAfterDESCRIBE()的最后调用了setupStreams（），那么就来探索一下setupStreams():

[cpp] view
plaincopy

void setupStreams() {

static MediaSubsessionIterator* setupIter = NULL;

if (setupIter == NULL) setupIter = new MediaSubsessionIterator(*session);

while ((subsession = setupIter->next()) != NULL) {

// We have another subsession left to set up:

if (subsession->clientPortNum() == 0) continue; // port # was not set

setupSubsession(subsession, streamUsingTCP, continueAfterSETUP);

return;

}

// We're done setting up subsessions.

delete setupIter;

if (!madeProgress) shutdown();

// Create output files:

if (createReceivers) {

if (outputQuickTimeFile) {

// Create a "QuickTimeFileSink", to write to 'stdout':

qtOut = QuickTimeFileSink::createNew(*env, *session, "stdout",

fileSinkBufferSize,

movieWidth, movieHeight,

movieFPS,

packetLossCompensate,

syncStreams,

generateHintTracks,

generateMP4Format);

if (qtOut == NULL) {

*env << "Failed to create QuickTime file sink for stdout: " << env->getResultMsg();

shutdown();

}

qtOut->startPlaying(sessionAfterPlaying, NULL);

} else if (outputAVIFile) {

// Create an "AVIFileSink", to write to 'stdout':

aviOut = AVIFileSink::createNew(*env, *session, "stdout",

fileSinkBufferSize,

movieWidth, movieHeight,

movieFPS,

packetLossCompensate);

if (aviOut == NULL) {

*env << "Failed to create AVI file sink for stdout: " << env->getResultMsg();

shutdown();

}

aviOut->startPlaying(sessionAfterPlaying, NULL);

} else {

// Create and start "FileSink"s for each subsession:

madeProgress = False;

MediaSubsessionIterator iter(*session);

while ((subsession = iter.next()) != NULL) {

if (subsession->readSource() == NULL) continue; // was not initiated

// Create an output file for each desired stream:

char outFileName[1000];

if (singleMedium == NULL) {

// Output file name is

// "<filename-prefix><medium_name>-<codec_name>-<counter>"

static unsigned streamCounter = 0;

snprintf(outFileName, sizeof outFileName, "%s%s-%s-%d",

fileNamePrefix, subsession->mediumName(),

subsession->codecName(), ++streamCounter);

} else {

sprintf(outFileName, "stdout");

}

FileSink* fileSink;

if (strcmp(subsession->mediumName(), "audio") == 0 &&

(strcmp(subsession->codecName(), "AMR") == 0 ||

strcmp(subsession->codecName(), "AMR-WB") == 0)) {

// For AMR audio streams, we use a special sink that inserts AMR frame hdrs:

fileSink = AMRAudioFileSink::createNew(*env, outFileName,

fileSinkBufferSize, oneFilePerFrame);

} else if (strcmp(subsession->mediumName(), "video") == 0 &&

(strcmp(subsession->codecName(), "H264") == 0)) {

// For H.264 video stream, we use a special sink that insert start_codes:

fileSink = H264VideoFileSink::createNew(*env, outFileName,

subsession->fmtp_spropparametersets(),

fileSinkBufferSize, oneFilePerFrame);

} else {

// Normal case:

fileSink = FileSink::createNew(*env, outFileName,

fileSinkBufferSize, oneFilePerFrame);

}

subsession->sink = fileSink;

if (subsession->sink == NULL) {

*env << "Failed to create FileSink for \"" << outFileName

<< "\": " << env->getResultMsg() << "\n";

} else {

if (singleMedium == NULL) {

*env << "Created output file: \"" << outFileName << "\"\n";

} else {

*env << "Outputting data from the \"" << subsession->mediumName()

<< "/" << subsession->codecName()

<< "\" subsession to 'stdout'\n";

}

if (strcmp(subsession->mediumName(), "video") == 0 &&

strcmp(subsession->codecName(), "MP4V-ES") == 0 &&

subsession->fmtp_config() != NULL) {

// For MPEG-4 video RTP streams, the 'config' information

// from the SDP description contains useful VOL etc. headers.

// Insert this data at the front of the output file:

unsigned configLen;

unsigned char* configData

= parseGeneralConfigStr(subsession->fmtp_config(), configLen);

struct timeval timeNow;

gettimeofday(&timeNow, NULL);

fileSink->addData(configData, configLen, timeNow);

delete[] configData;

}

subsession->sink->startPlaying(*(subsession->readSource()),

subsessionAfterPlaying,

subsession);

// Also set a handler to be called if a RTCP "BYE" arrives

// for this subsession:

if (subsession->rtcpInstance() != NULL) {

subsession->rtcpInstance()->setByeHandler(subsessionByeHandler, subsession);

}

madeProgress = True;

}

}

if (!madeProgress) shutdown();

}

}

// Finally, start playing each subsession, to start the data flow:

if (duration == 0) {

if (scale > 0) duration = session->playEndTime() - initialSeekTime; // use SDP end time

else if (scale < 0) duration = initialSeekTime;

}

if (duration < 0) duration = 0.0;

endTime = initialSeekTime;

if (scale > 0) {

if (duration <= 0) endTime = -1.0f;

else endTime = initialSeekTime + duration;

} else {

endTime = initialSeekTime - duration;

if (endTime < 0) endTime = 0.0f;

}

startPlayingSession(session, initialSeekTime, endTime, scale, continueAfterPLAY);

}

[cpp] view
plaincopy

fileSink = H264VideoFileSink::createNew(*env, outFileName,

subsession->fmtp_spropparametersets(),

fileSinkBufferSize, oneFilePerFrame);

然后是subsession->sink = fileSink;

然后比较关键的是就是

[cpp] view
plaincopy

subsession->sink->startPlaying(*(subsession->readSource()),

subsessionAfterPlaying,

subsession);

我们来看看这个startPlaying

[cpp] view
plaincopy

Boolean MediaSink::startPlaying(MediaSource& source,

afterPlayingFunc* afterFunc,

void* afterClientData) {

// Make sure we're not already being played:

if (fSource != NULL) {

envir().setResultMsg("This sink is already being played");

return False;

}

// Make sure our source is compatible:

if (!sourceIsCompatibleWithUs(source)) {

envir().setResultMsg("MediaSink::startPlaying(): source is not compatible!");

return False;

}

fSource = (FramedSource*)&source;

fAfterFunc = afterFunc;

fAfterClientData = afterClientData;

return continuePlaying();

}

上面中subsession->readSource()返回的是fReadSource就是在createSourceObjects()中建立的那个source。我们看到这里赋值给了fSource。

continuePlaying()在MediaSink中为纯虚函数，在FileSink中有定义。

[cpp] view
plaincopy

Boolean FileSink::continuePlaying() {

if (fSource == NULL) return False;

fSource->getNextFrame(fBuffer, fBufferSize,

afterGettingFrame, this,

onSourceClosure, this);

return True;

}

其实很简单就是fSource的getNextFrame。这里的fSource就是MediaSink中的fSource。就是H264VideoRTPSource。

所有的getNextFrame都一样就是FrameSource中的getNextFrame。把fBuffer给fTo，fBufferSize就是fMaxSize。

我们来看看这个fBuffer，

[cpp] view
plaincopy

fBuffer = new unsigned char[bufferSize];

在

[cpp] view
plaincopy

fileSink = H264VideoFileSink::createNew(*env, outFileName,

subsession->fmtp_spropparametersets(),

fileSinkBufferSize, oneFilePerFrame);

中fileSinkBufferSize是100000。

getNextFrame之后执行的是doGetNextFrame(),一般在子类里面实现。H264VideoRTPSource中没有实现，但在他的父类MultiFramedRTPSource里面有实现

[cpp] view
plaincopy

void MultiFramedRTPSource::doGetNextFrame() {

if (!fAreDoingNetworkReads) {

// Turn on background read handling of incoming packets:

fAreDoingNetworkReads = True;

TaskScheduler::BackgroundHandlerProc* handler

= (TaskScheduler::BackgroundHandlerProc*)&networkReadHandler;

fRTPInterface.startNetworkReading(handler);

}

fSavedTo = fTo;

fSavedMaxSize = fMaxSize;

fFrameSize = 0; // for now

fNeedDelivery = True;

doGetNextFrame1();

}