VVC代码学习2：帧内亮度预测函数estIntraPredLumaQT

一、estIntraPredLumaQT调用结构

xCheckRDCostIntra：帧内预测的入口函数，会调用estIntraPredLumaQT进行亮度的预测，调用estIntraPredChromaQT进行色度的预测。

estIntraPredLumaQT：亮度预测的入口函数，完成亮度的预测，角度模式的选择等操作。

preIntraAng：进行角度预测，获得预测像素值。

xIntraCodingLumaISP：ISP模式下的完整RDO过程，ISP模式下会将当前CU划分为4个或2个TU，每个TU分别经过变换，量化，反变换，反量化，重建等操作进行完整的RDO过程，计算出完整的率失真代价。

xRecurIntraCodingLumaQT：普通角度模式下的RDO过程。

二、函数流程

VTM的亮度预测函数和HM中大致流程是差不多的，主要分为三个阶段：

1.RMD过程：为了减少运算复杂度，直接对残差进行哈达玛变换求得SATD失真计算代价，将最小代价前几的模式加入全率失真模式列表中。

2.MPM：根据相邻块推导出当前块的最可能模式，加入到全率失真模式列表中

3.RDO：遍历全率失真模式列表中的模式，进行完整的变换、量化、反变换、反量化等操作，算出失真和比特求出RD代价，选择代价最小的模式作为当前块的最优角度模式。

但是在VTM中又新增了32种帧内角度模式，引入了许多新工具，所以每个阶段相较于HM都变得很复杂。具体的流程直接看函数：

bool IntraSearch::estIntraPredLumaQT( CodingUnit &cu, Partitioner &partitioner, const double bestCostSoFar, bool mtsCheckRangeFlag, int mtsFirstCheckId, int mtsLastCheckId, bool moreProbMTSIdxFirst )
{
CodingStructure       &cs            = *cu.cs;//父级CS
const SPS             &sps           = *cs.sps;//序列参数集
const uint32_t             uiWidthBit    = floorLog2(partitioner.currArea().lwidth() );
const uint32_t             uiHeightBit   =                   floorLog2(partitioner.currArea().lheight());

// Lambda calculation at equivalent Qp of 4 is recommended because at that Qp, the quantization divisor is 1.
const double sqrtLambdaForFirstPass = m_pcRdCost->getMotionLambda(cu.transQuantBypass) * FRAC_BITS_SCALE;

//===== loop over partitions =====
//更新上下文信息
const TempCtx ctxStart          ( m_CtxCache, m_CABACEstimator->getCtx() );
const TempCtx ctxStartMipFlag    ( m_CtxCache, SubCtx( Ctx::MipFlag,          m_CABACEstimator->getCtx() ) );
#if !JVET_O0925_MIP_SIMPLIFICATIONS
const TempCtx ctxStartMipMode    ( m_CtxCache, SubCtx( Ctx::MipMode,          m_CABACEstimator->getCtx() ) );
#endif
const TempCtx ctxStartIspMode    ( m_CtxCache, SubCtx( Ctx::ISPMode,          m_CABACEstimator->getCtx() ) );
const TempCtx ctxStartPlanarFlag ( m_CtxCache, SubCtx( Ctx::IntraLumaPlanarFlag, m_CABACEstimator->getCtx() ) );
const TempCtx ctxStartIntraMode(m_CtxCache, SubCtx(Ctx::IntraLumaMpmFlag, m_CABACEstimator->getCtx()));
const TempCtx ctxStartMrlIdx      ( m_CtxCache, SubCtx( Ctx::MultiRefLineIdx,        m_CABACEstimator->getCtx() ) );

CHECK( !cu.firstPU, "CU has no PUs" );
const bool keepResi   = cs.pps->getPpsRangeExtension().getCrossComponentPredictionEnabledFlag() || KEEP_PRED_AND_RESI_SIGNALS;

// variables for saving fast intra modes scan results across multiple LFNST passes
bool LFNSTLoadFlag = sps.getUseLFNST() && cu.lfnstIdx != 0;
bool LFNSTSaveFlag = sps.getUseLFNST() && cu.lfnstIdx == 0;

LFNSTSaveFlag &= sps.getUseIntraMTS() ? cu.mtsFlag == 0 : true;

const uint32_t lfnstIdx = cu.lfnstIdx;
#if JVET_O0050_LOCAL_DUAL_TREE
double costInterCU = findInterCUCost( cu );
#endif

const int width  = partitioner.currArea().lwidth();//宽
const int height = partitioner.currArea().lheight();//高

// Marking MTS usage for faster MTS
// 0: MTS is either not applicable for current CU (cuWidth > MTS_INTRA_MAX_CU_SIZE or cuHeight > MTS_INTRA_MAX_CU_SIZE), not active in the config file or the fast decision algorithm is not used in this case
// 1: MTS fast algorithm can be applied for the current CU, and the DCT2 is being checked
// 2: MTS is being checked for current CU. Stored results of DCT2 can be utilized for speedup
// 当mtsUsageFlag为0时不使用MTS；为1时使用MTS快速算法，首先检测DCT2变换；为2时使用DCT2变换的结果加速后续的变换核
uint8_t mtsUsageFlag = 0;
const int maxSizeEMT = MTS_INTRA_MAX_CU_SIZE;//能使用MTS的最大尺寸32
if( width <= maxSizeEMT && height <= maxSizeEMT && sps.getUseIntraMTS() )
{
mtsUsageFlag = ( sps.getUseLFNST() && cu.mtsFlag == 1 ) ? 2 : 1;
}

if( width * height < 64 && !m_pcEncCfg->getUseFastLFNST() )
{
mtsUsageFlag = 0;
}

double bestCurrentCost = bestCostSoFar;//当前最优代价
#if MAX_TB_SIZE_SIGNALLING
bool testISP = sps.getUseISP() && cu.mtsFlag == 0 && cu.lfnstIdx == 0 && CU::canUseISP( width, height, cu.cs->sps->getMaxTbSize() );//是否进行ISP
#else
bool testISP = sps.getUseISP() && cu.mtsFlag == 0 && cu.lfnstIdx == 0 && CU::canUseISP( width, height, MAX_TB_SIZEY );
#endif
#if !JVET_O0502_ISP_CLEANUP
bool ispHorIsFirstTest = testISP ? CU::firstTestISPHorSplit( width, height, COMPONENT_Y, nullptr, nullptr ) : true;
int ispOptions[] = { NOT_INTRA_SUBPARTITIONS, HOR_INTRA_SUBPARTITIONS, VER_INTRA_SUBPARTITIONS };
if ( !ispHorIsFirstTest )
{
ispOptions[1] = VER_INTRA_SUBPARTITIONS;
ispOptions[2] = HOR_INTRA_SUBPARTITIONS;
}
#endif
if( testISP )
{
#if JVET_O0502_ISP_CLEANUP
//reset the variables used for the tests
m_ispCandListHor.clear();//ISP水平划分列表
m_ispCandListVer.clear();//ISP垂直划分列表
m_regIntraRDListWithCosts.clear();
m_ispTestedModes.clear();//ISP已测试列表
//save the number of subpartitions
m_ispTestedModes.numTotalParts[0] = (int)height >> floorLog2(CU::getISPSplitDim(width, height, TU_1D_HORZ_SPLIT));
m_ispTestedModes.numTotalParts[1] = (int)width >> floorLog2(CU::getISPSplitDim(width, height, TU_1D_VERT_SPLIT));
#else
//variables for the full RD list without MRL modes
m_rdModeListWithoutMrl      .clear();
m_rdModeListWithoutMrlHor   .clear();
m_rdModeListWithoutMrlVer   .clear();
//variables with data from regular intra used to skip ISP splits
m_intraModeDiagRatio        .clear();
m_intraModeHorVerRatio      .clear();
m_intraModeTestedNormalIntra.clear();
#endif
}

#if JVET_O1136_TS_BDPCM_SIGNALLING
const bool testBDPCM = sps.getBDPCMEnabledFlag() && CU::bdpcmAllowed( cu, ComponentID( partitioner.chType ) ) && cu.mtsFlag == 0 && cu.lfnstIdx == 0;
#else
const bool testBDPCM = m_pcEncCfg->getRDPCM() && CU::bdpcmAllowed( cu, ComponentID( partitioner.chType ) ) && cu.mtsFlag == 0 && cu.lfnstIdx == 0;
#endif
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiHadModeList;//哈达玛模式列表
static_vector<double, FAST_UDI_MAX_RDMODE_NUM> CandCostList;//率失真代价列表
static_vector<double, FAST_UDI_MAX_RDMODE_NUM> CandHadList;//哈达玛失真列表

auto &pu = *cu.firstPU;//获取PU
bool validReturn = false;
{
CandHadList.clear();
CandCostList.clear();
uiHadModeList.clear();

CHECK(pu.cu != &cu, "PU is not contained in the CU");

//===== determine set of modes to be tested (using prediction signal only) =====
int numModesAvailable = NUM_LUMA_MODE; // total number of Intra modes 67个帧内模式
const bool fastMip    = sps.getUseMIP() && m_pcEncCfg->getUseFastMIP();//快速MIP
#if JVET_O0925_MIP_SIMPLIFICATIONS
#if JVET_O0545_MAX_TB_SIGNALLING
const bool mipAllowed = sps.getUseMIP() && isLuma(partitioner.chType) && pu.lwidth() <= cu.cs->sps->getMaxTbSize() && pu.lheight() <= cu.cs->sps->getMaxTbSize() && ((cu.lfnstIdx == 0) || allowLfnstWithMip(cu.firstPU->lumaSize()));
#else
const bool mipAllowed = sps.getUseMIP() && isLuma(partitioner.chType) && pu.lwidth() <= MIP_MAX_WIDTH && pu.lheight() <= MIP_MAX_HEIGHT && ((cu.lfnstIdx == 0) || allowLfnstWithMip(cu.firstPU->lumaSize()));
#endif
const bool testMip    = mipAllowed && mipModesAvailable(pu.Y());//标记是否加入MIP模式
#else
#if JVET_O0545_MAX_TB_SIGNALLING
const bool mipAllowed = sps.getUseMIP() && ( cu.lfnstIdx == 0 ) && isLuma( partitioner.chType ) && pu.lwidth() <= cu.cs->sps->getMaxTbSize() && pu.lheight() <= cu.cs->sps->getMaxTbSize();
#else
const bool mipAllowed = sps.getUseMIP() && ( cu.lfnstIdx == 0 ) && isLuma( partitioner.chType ) && pu.lwidth() <= MIP_MAX_WIDTH && pu.lheight() <= MIP_MAX_HEIGHT;
#endif
const bool testMip    = mipAllowed && mipModesAvailable( pu.Y() ) && !(fastMip && (cu.lwidth() > 2 * cu.lheight() || cu.lheight() > 2 * cu.lwidth()));
#endif

static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiRdModeList;//全率失真模式列表

int numModesForFullRD = 3;
numModesForFullRD = g_aucIntraModeNumFast_UseMPM_2D[uiWidthBit - MIN_CU_LOG2][uiHeightBit - MIN_CU_LOG2];//根据块的大小决定进行全率失真的个数

#if INTRA_FULL_SEARCH
numModesForFullRD = numModesAvailable;
#endif

if( mtsUsageFlag != 2 )
{
// this should always be true
CHECK( !pu.Y().valid(), "PU is not valid" );
#if ENABLE_JVET_L0283_MRL
bool isFirstLineOfCtu = (((pu.block(COMPONENT_Y).y)&((pu.cs->sps)->getMaxCUWidth() - 1)) == 0);//标记是否是CTU的第一行，用来决定是否进行MRL
int numOfPassesExtendRef = (isFirstLineOfCtu ? 1 : MRL_NUM_REF_LINES);//如果不是CTU的第一行，多参考行的候选数为3
#endif
pu.multiRefIdx = 0;

if( numModesForFullRD != numModesAvailable )
{
CHECK( numModesForFullRD >= numModesAvailable, "Too many modes for full RD search" );

const CompArea &area = pu.Y();

PelBuf piOrg         = cs.getOrgBuf(area);//原始像素
PelBuf piPred        = cs.getPredBuf(area);//预测像素

DistParam distParamSad;//SAD失真
DistParam distParamHad;//HAD（哈达玛）失真
if (cu.slice->getLmcsEnabledFlag() && m_pcReshape->getCTUFlag())
{
CompArea      tmpArea(COMPONENT_Y, area.chromaFormat, Position(0, 0), area.size());
PelBuf tmpOrg = m_tmpStorageLCU.getBuf(tmpArea);
tmpOrg.copyFrom(piOrg);
tmpOrg.rspSignal(m_pcReshape->getFwdLUT());
m_pcRdCost->setDistParam(distParamSad, tmpOrg, piPred, sps.getBitDepth(CHANNEL_TYPE_LUMA), COMPONENT_Y, false); // Use SAD cost
m_pcRdCost->setDistParam(distParamHad, tmpOrg, piPred, sps.getBitDepth(CHANNEL_TYPE_LUMA), COMPONENT_Y,  true); // Use HAD (SATD) cost
}
else
{
m_pcRdCost->setDistParam(distParamSad, piOrg, piPred, sps.getBitDepth(CHANNEL_TYPE_LUMA), COMPONENT_Y, false); // Use SAD cost 使用SAD
m_pcRdCost->setDistParam(distParamHad, piOrg, piPred, sps.getBitDepth(CHANNEL_TYPE_LUMA), COMPONENT_Y,  true); // Use HAD (SATD) cost 使用SATD
}

distParamSad.applyWeight = false;
distParamHad.applyWeight = false;

if( testMip)//如果使用MIP模式
{
#if JVET_O0925_MIP_SIMPLIFICATIONS
numModesForFullRD += fastMip? std::max(numModesForFullRD, floorLog2(std::min(pu.lwidth(), pu.lheight())) - 1) : numModesForFullRD;//调整全率失真个数
#else
numModesForFullRD += fastMip? std::max(2, floorLog2(std::min(pu.lwidth(), pu.lheight())) - 1) : numModesForFullRD;
#endif
}
const int numHadCand = (testMip ? 2 : 1) * 3;//哈达玛变换个数

//*** Derive (regular) candidates using Hadamard
cu.mipFlag = false;

//===== init pattern for luma prediction =====
initIntraPatternChType(cu, pu.Y(), true);//参考像素的准备，滤波等操作
bool bSatdChecked[NUM_INTRA_MODE];//标记是否被检测过
memset( bSatdChecked, 0, sizeof( bSatdChecked ) );

if( !LFNSTLoadFlag )
{
for( int modeIdx = 0; modeIdx < numModesAvailable; modeIdx++ ) //第一轮初选，遍历HEVC中规定的35种角度模式
{
uint32_t       uiMode = modeIdx;
Distortion minSadHad = 0;

// Skip checking extended Angular modes in the first round of SATD 跳过VVC新增的帧内模式
if( uiMode > DC_IDX && ( uiMode & 1 ) )
{
continue;
}

bSatdChecked[uiMode] = true;

pu.intraDir[0] = modeIdx;

initPredIntraParams(pu, pu.Y(), sps);
if( useDPCMForFirstPassIntraEstimation( pu, uiMode ) )
{
encPredIntraDPCM( COMPONENT_Y, piOrg, piPred, uiMode );
}
else
{
predIntraAng( COMPONENT_Y, piPred, pu);//角度预测的入口函数
}
// Use the min between SAD and HAD as the cost criterion

20000
// SAD is scaled by 2 to align with the scaling of HAD
minSadHad += std::min(distParamSad.distFunc(distParamSad)*2, distParamHad.distFunc(distParamHad));//使用SAD和HAD最小值

// NB xFracModeBitsIntra will not affect the mode for chroma that may have already been pre-estimated.
m_CABACEstimator->getCtx() = SubCtx( Ctx::MipFlag, ctxStartMipFlag );
m_CABACEstimator->getCtx() = SubCtx( Ctx::ISPMode, ctxStartIspMode );
m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaPlanarFlag, ctxStartPlanarFlag);
m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaMpmFlag, ctxStartIntraMode);
m_CABACEstimator->getCtx() = SubCtx( Ctx::MultiRefLineIdx, ctxStartMrlIdx );

uint64_t fracModeBits = xFracModeBitsIntra(pu, uiMode, CHANNEL_TYPE_LUMA);//比特数

double cost = ( double ) minSadHad + (double)fracModeBits * sqrtLambdaForFirstPass;//代价

DTRACE(g_trace_ctx, D_INTRA_COST, "IntraHAD: %u, %llu, %f (%d)\n", minSadHad, fracModeBits, cost, uiMode);

updateCandList( ModeInfo(false, 0, NOT_INTRA_SUBPARTITIONS, uiMode), cost,          uiRdModeList,  CandCostList, numModesForFullRD );//更新列表
updateCandList( ModeInfo(false, 0, NOT_INTRA_SUBPARTITIONS, uiMode), (double)minSadHad, uiHadModeList, CandHadList,  numHadCand );//更新列表
}
#if JVET_O0925_MIP_SIMPLIFICATIONS
if( !sps.getUseMIP() && LFNSTSaveFlag )
#else
if( LFNSTSaveFlag )
#endif
{
// save found best modes
m_uiSavedNumRdModesLFNST   = numModesForFullRD;
m_uiSavedRdModeListLFNST   = uiRdModeList;
m_dSavedModeCostLFNST      = CandCostList;
// PBINTRA fast
m_uiSavedHadModeListLFNST  = uiHadModeList;
m_dSavedHadListLFNST       = CandHadList;
LFNSTSaveFlag              = false;
}
} // NSSTFlag
#if JVET_O0925_MIP_SIMPLIFICATIONS
if( !sps.getUseMIP() && LFNSTLoadFlag )
#else
else
#endif
{
// restore saved modes
numModesForFullRD = m_uiSavedNumRdModesLFNST;
uiRdModeList      = m_uiSavedRdModeListLFNST;
CandCostList      = m_dSavedModeCostLFNST;
// PBINTRA fast
uiHadModeList     = m_uiSavedHadModeListLFNST;
CandHadList       = m_dSavedHadListLFNST;
#if !JVET_O0925_MIP_SIMPLIFICATIONS

LFNSTLoadFlag     = false;
#endif
} // !LFNSTFlag

#if JVET_O0925_MIP_SIMPLIFICATIONS
if (!(sps.getUseMIP() && LFNSTLoadFlag))
{
#else
CHECK( uiRdModeList.size() != numModesForFullRD, "Error: RD mode list size" );
#endif
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> parentCandList = uiRdModeList;//第一轮初选列表赋给parentCandList

// Second round of SATD for extended Angular modes  第二轮粗选，对第一轮粗选加入的模式左邻和右邻模式计算代价并更新列表
for (int modeIdx = 0; modeIdx < numModesForFullRD; modeIdx++)
{
unsigned parentMode = parentCandList[modeIdx].modeId;
if (parentMode > (DC_IDX + 1) && parentMode < (NUM_LUMA_MODE - 1))
{
for (int subModeIdx = -1; subModeIdx <= 1; subModeIdx += 2)
{
unsigned mode = parentMode + subModeIdx;

if (!bSatdChecked[mode])//如果没被检测过
{
pu.intraDir[0] = mode;

initPredIntraParams(pu, pu.Y(), sps);
if (useDPCMForFirstPassIntraEstimation(pu, mode))
{
encPredIntraDPCM(COMPONENT_Y, piOrg, piPred, mode);
}
else
{
predIntraAng(COMPONENT_Y, piPred, pu );
}

// Use the min between SAD and SATD as the cost criterion
// SAD is scaled by 2 to align with the scaling of HAD
Distortion minSadHad = std::min(distParamSad.distFunc(distParamSad)*2, distParamHad.distFunc(distParamHad));

// NB xFracModeBitsIntra will not affect the mode for chroma that may have already been pre-estimated.
m_CABACEstimator->getCtx() = SubCtx( Ctx::MipFlag, ctxStartMipFlag );
m_CABACEstimator->getCtx() = SubCtx( Ctx::ISPMode, ctxStartIspMode );
m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaPlanarFlag, ctxStartPlanarFlag);
m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaMpmFlag, ctxStartIntraMode);
m_CABACEstimator->getCtx() = SubCtx( Ctx::MultiRefLineIdx, ctxStartMrlIdx );

uint64_t fracModeBits = xFracModeBitsIntra(pu, mode, CHANNEL_TYPE_LUMA);

double cost = (double) minSadHad + (double) fracModeBits * sqrtLambdaForFirstPass;

updateCandList( ModeInfo( false, 0, NOT_INTRA_SUBPARTITIONS, mode ), cost,        uiRdModeList,  CandCostList, numModesForFullRD );
updateCandList( ModeInfo( false, 0, NOT_INTRA_SUBPARTITIONS, mode ), (double)minSadHad, uiHadModeList, CandHadList,  numHadCand );

bSatdChecked[mode] = true;
}
}
}
}
if ( testISP )//如果使用ISP
{
#if JVET_O0502_ISP_CLEANUP
// we save the regular intra modes list
m_ispCandListHor = uiRdModeList;//把全率失真列表的模式赋值给水平ISP列表
#else
//we save the list with no mrl modes to keep only the Hadamard selected modes (no mpms)
m_rdModeListWithoutMrl = uiRdModeList;
#endif
}
#if ENABLE_JVET_L0283_MRL
//MRL下的MPM列表建立
pu.multiRefIdx = 1;
const int  numMPMs = NUM_MOST_PROBABLE_MODES;
unsigned  multiRefMPM [numMPMs];
PU::getIntraMPMs(pu, multiRefMPM);//获取MPM列表
for (int mRefNum = 1; mRefNum < numOfPassesExtendRef; mRefNum++)
{
int multiRefIdx = MULTI_REF_LINE_IDX[mRefNum];//1 or 3

pu.multiRefIdx = multiRefIdx;
{
initIntraPatternChType(cu, pu.Y(), true);
}
for (int x = 1; x < numMPMs; x++)//遍历MPM模式列表，更新列表
{
uint32_t mode = multiRefMPM[x];
{
pu.intraDir[0] = mode;
initPredIntraParams(pu, pu.Y(), sps);

if (useDPCMForFirstPassIntraEstimation(pu, mode))
{
encPredIntraDPCM(COMPONENT_Y, piOrg, piPred, mode);
}
else
{
predIntraAng(COMPONENT_Y, piPred, pu);
}

// Use the min between SAD and SATD as the cost criterion
// SAD is scaled by 2 to align with the scaling of HAD
Distortion minSadHad = std::min(distParamSad.distFunc(distParamSad)*2, distParamHad.distFunc(distParamHad));

// NB xFracModeBitsIntra will not affect the mode for chroma that may have already been pre-estimated.
m_CABACEstimator->getCtx() = SubCtx( Ctx::MipFlag, ctxStartMipFlag );
m_CABACEstimator->getCtx() = SubCtx( Ctx::ISPMode, ctxStartIspMode );
m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaPlanarFlag, ctxStartPlanarFlag);
m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaMpmFlag, ctxStartIntraMode);
m_CABACEstimator->getCtx() = SubCtx( Ctx::MultiRefLineIdx, ctxStartMrlIdx );

uint64_t fracModeBits = xFracModeBitsIntra(pu, mode, CHANNEL_TYPE_LUMA);

double cost = (double)minSadHad + (double)fracModeBits * sqrtLambdaForFirstPass;
updateCandList( ModeInfo( false, multiRefIdx, NOT_INTRA_SUBPARTITIONS, mode ), cost,        uiRdModeList,  CandCostList, numModesForFullRD );
updateCandList( ModeInfo( false, multiRefIdx, NOT_INTRA_SUBPARTITIONS, mode ), (double)minSadHad, uiHadModeList, CandHadList,  numHadCand );
}
}
}
#endif
CHECKD( uiRdModeList.size() != numModesForFullRD, "Error: RD mode list size" );

#if JVET_O0925_MIP_SIMPLIFICATIONS
if (LFNSTSaveFlag && testMip && !allowLfnstWithMip(cu.firstPU->lumaSize())) // save a different set for the next run
{
// save found best modes
m_uiSavedRdModeListLFNST = uiRdModeList;
m_dSavedModeCostLFNST = CandCostList;
// PBINTRA fast
m_uiSavedHadModeListLFNST = uiHadModeList;
m_dSavedHadListLFNST = CandHadList;
m_uiSavedNumRdModesLFNST = g_aucIntraModeNumFast_UseMPM_2D[uiWidthBit - MIN_CU_LOG2][uiHeightBit - MIN_CU_LOG2];
m_uiSavedRdModeListLFNST.resize(m_uiSavedNumRdModesLFNST);
m_dSavedModeCostLFNST.resize(m_uiSavedNumRdModesLFNST);
// PBINTRA fast
m_uiSavedHadModeListLFNST.resize(3);
m_dSavedHadListLFNST.resize(3);
LFNSTSaveFlag = false;
}
#endif
//*** Derive MIP candidates using Hadamard
if (testMip) //MIP模式候选集
{
cu.mipFlag = true;
pu.multiRefIdx = 0;//多参考索引标志位

#if JVET_O0925_MIP_SIMPLIFICATIONS
double mipHadCost[MAX_NUM_MIP_MODE] = { MAX_DOUBLE };

initIntraPatternChType(cu, pu.Y());
#endif
initIntraMip( pu );

for (uint32_t uiMode = 0; uiMode < getNumModesMip(pu.Y()); uiMode++)
{
pu.intraDir[CHANNEL_TYPE_LUMA] = uiMode;
predIntraMip(COMPONENT_Y, piPred, pu);//MIP模式的预测

// Use the min between SAD and HAD as the cost criterion
// SAD is scaled by 2 to align with the scaling of HAD
Distortion minSadHad = std::min(distParamSad.distFunc(distParamSad)*2, distParamHad.distFunc(distParamHad));

m_CABACEstimator->getCtx() = SubCtx( Ctx::MipFlag, ctxStartMipFlag );
#if !JVET_O0925_MIP_SIMPLIFICATIONS
m_CABACEstimator->getCtx() = SubCtx( Ctx::MipMode, ctxStartMipMode );
#endif

uint64_t fracModeBits = xFracModeBitsIntra(pu, uiMode, CHANNEL_TYPE_LUMA);

double cost = double(minSadHad) + double(fracModeBits) * sqrtLambdaForFirstPass;
#if JVET_O0925_MIP_SIMPLIFICATIONS
mipHadCost[uiMode] = cost;
DTRACE(g_trace_ctx, D_INTRA_COST, "IntraMIP: %u, %llu, %f (%d)\n", minSadHad, fracModeBits, cost, uiMode);

updateCandList(ModeInfo(true, 0, NOT_INTRA_SUBPARTITIONS, uiMode), cost, uiRdModeList, CandCostList, numModesForFullRD + 1);
updateCandList(ModeInfo(true, 0, NOT_INTRA_SUBPARTITIONS, uiMode), 0.8*double(minSadHad), uiHadModeList, CandHadList, numHadCand);
#else
updateCandList(ModeInfo(true, 0, NOT_INTRA_SUBPARTITIONS, uiMode),        cost,  uiRdModeList,  CandCostList, numModesForFullRD);
updateCandList(ModeInfo(true, 0, NOT_INTRA_SUBPARTITIONS, uiMode), double(minSadHad), uiHadModeList, CandHadList, numHadCand);
#endif
}

const double thresholdHadCost = 1.0 + 1.4 / sqrt((double)(pu.lwidth()*pu.lheight()));
#if JVET_O0925_MIP_SIMPLIFICATIONS
reduceHadCandList(uiRdModeList, CandCostList, numModesForFullRD, thresholdHadCost, mipHadCost, pu, fastMip);
#else
reduceHadCandList(uiRdModeList, CandCostList, numModesForFullRD, thresholdHadCost, 0.0);
#endif
}
#if JVET_O0925_MIP_SIMPLIFICATIONS
if ( sps.getUseMIP() && LFNSTSaveFlag)
{
// save found best modes
m_uiSavedNumRdModesLFNST = numModesForFullRD;
m_uiSavedRdModeListLFNST = uiRdModeList;
m_dSavedModeCostLFNST = CandCostList;
// PBINTRA fast
m_uiSavedHadModeListLFNST = uiHadModeList;
m_dSavedHadListLFNST = CandHadList;
LFNSTSaveFlag = false;
}
}
else //if( sps.getUseMIP() && LFNSTLoadFlag)
{
// restore saved modes
numModesForFullRD = m_uiSavedNumRdModesLFNST;
uiRdModeList = m_uiSavedRdModeListLFNST;
CandCostList = m_dSavedModeCostLFNST;
// PBINTRA fast
uiHadModeList = m_uiSavedHadModeListLFNST;
CandHadList = m_dSavedHadListLFNST;
}
#endif
//普通的MPM模式
if( m_pcEncCfg->getFastUDIUseMPMEnabled() )
{
const int numMPMs = NUM_MOST_PROBABLE_MODES;//6
unsigned  uiPreds[numMPMs];

pu.multiRefIdx = 0;//多参考索引0

const int numCand = PU::getIntraMPMs( pu, uiPreds );//获取MPM模式列表返回个数

for( int j = 0; j < numCand; j++ )//如果uiRdModeList不存在则加入
{
bool mostProbableModeIncluded = false;
ModeInfo mostProbableMode( false, 0, NOT_INTRA_SUBPARTITIONS, uiPreds[j] );

for( int i = 0; i < numModesForFullRD; i++ )
{
mostProbableModeIncluded |= ( mostProbableMode == uiRdModeList[i] );
}
if( !mostProbableModeIncluded )
{
numModesForFullRD++;
uiRdModeList.push_back( mostProbableMode );
CandCostList.push_back(0);
}
}
if ( testISP )//把MPM模式加入水平ISP模式列表
{
#if JVET_O0502_ISP_CLEANUP
// we add the MPMs to the list that contains only regular intra modes
for (int j = 0; j < numCand; j++)
{
bool     mostProbableModeIncluded = false;
ModeInfo mostProbableMode(false, 0, NOT_INTRA_SUBPARTITIONS, uiPreds[j]);

for (int i = 0; i < m_ispCandListHor.size(); i++)
{
mostProbableModeIncluded |= (mostProbableMode == m_ispCandListHor[i]);
}
if (!mostProbableModeIncluded)
{
m_ispCandListHor.push_back(mostProbableMode);
}
}
#else
//we add the ISP MPMs to the list without mrl modes
m_rdModeListWithoutMrlHor = m_rdModeListWithoutMrl;
m_rdModeListWithoutMrlVer = m_rdModeListWithoutMrl;
for (int k = 0; k < m_rdModeListWithoutMrl.size(); k++)
{
m_rdModeListWithoutMrlHor[k].ispMod = HOR_INTRA_SUBPARTITIONS;
m_rdModeListWithoutMrlVer[k].ispMod = VER_INTRA_SUBPARTITIONS;
}
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM>* listPointer;
for( int k = 1; k < NUM_INTRA_SUBPARTITIONS_MODES; k++ )
{
cu.ispMode = ispOptions[k];
listPointer = &( cu.ispMode == HOR_INTRA_SUBPARTITIONS ? m_rdModeListWithoutMrlHor : m_rdModeListWithoutMrlVer );
const int numCandISP = PU::getIntraMPMs( pu, uiPreds );
for( int j = 0; j < numCandISP; j++ )
{
bool mostProbableModeIncluded = false;
ModeInfo mostProbableMode( false, 0, cu.ispMode, uiPreds[j] );

for( int i = 0; i < listPointer->size(); i++ )
{
mostProbableModeIncluded |= ( mostProbableMode == listPointer->at( i ) );
}
if( !mostProbableModeIncluded )
{
listPointer->push_back( mostProbableMode );
}
}
}
cu.ispMode = NOT_INTRA_SUBPARTITIONS;
#endif
}
}
#if !JVET_O0925_MIP_SIMPLIFICATIONS
//*** Add MPMs for MIP to candidate list
if (!fastMip && testMip && pu.lwidth() < 8 && pu.lheight() < 8)
{
unsigned mpm[NUM_MPM_MIP];
int numCandMip = PU::getMipMPMs(pu, mpm);

for( int j = 0; j < numCandMip; j++ )
{
bool mostProbableModeIncluded = false;
ModeInfo mostProbableMode(true, 0, NOT_INTRA_SUBPARTITIONS, mpm[j]);
for( int i = 0; i < numModesForFullRD; i++ )
{
mostProbableModeIncluded |= (mostProbableMode == uiRdModeList[i]);
}
if( !mostProbableModeIncluded )
{
numModesForFullRD++;
uiRdModeList.push_back( mostProbableMode );
CandCostList.push_back(0);
}
}
}
#endif
}
else
{
THROW( "Full search not supported for MIP" );
}
if( sps.getUseLFNST() && mtsUsageFlag == 1 )
{
// Store the modes to be checked with RD
m_savedNumRdModes[ lfnstIdx ]     = numModesForFullRD;
std::copy_n( uiRdModeList.begin(),  numModesForFullRD, m_savedRdModeList[ lfnstIdx ] );
}
}
else //mtsUsage = 2 (here we potentially reduce the number of modes that will be full-RD checked)
{
if( ( m_pcEncCfg->getUseFastLFNST() || !cu.slice->isIntra() ) && m_bestModeCostValid[ lfnstIdx ] )
{
numModesForFullRD = 0;

double thresholdSkipMode = 1.0 + ( ( cu.lfnstIdx > 0 ) ? 0.1 : 1.0 ) * ( 1.4 / sqrt( ( double ) ( width*height ) ) );

// Skip checking the modes with much larger R-D cost than the best mode
for( int i = 0; i < m_savedNumRdModes[ lfnstIdx ]; i++ )
{
if( m_modeCostStore[ lfnstIdx ][ i ] <= thresholdSkipMode * m_bestModeCostStore[ lfnstIdx ] )
{
uiRdModeList.push_back( m_savedRdModeList[ lfnstIdx ][ i ] );
numModesForFullRD++;
}
}
}
else //this is necessary because we skip the candidates list calculation, since it was already obtained for the DCT-II. Now we load it
{
// Restore the modes to be checked with RD
numModesForFullRD = m_savedNumRdModes[ lfnstIdx ];
uiRdModeList.resize( numModesForFullRD );
std::copy_n( m_savedRdModeList[ lfnstIdx ], m_savedNumRdModes[ lfnstIdx ], uiRdModeList.begin() );
CandCostList.resize( numModesForFullRD );
}
}

#if !JVET_O0502_ISP_CLEANUP
if( testISP ) // we remove the non-MPMs from the ISP lists
{
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiRdModeListCopyHor = m_rdModeListWithoutMrlHor;
m_rdModeListWithoutMrlHor.clear();
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> uiRdModeListCopyVer = m_rdModeListWithoutMrlVer;
m_rdModeListWithoutMrlVer.clear();
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> *listPointerCopy, *listPointer;
for( int ispOptionIdx = 1; ispOptionIdx < NUM_INTRA_SUBPARTITIONS_MODES; ispOptionIdx++ )
{
cu.ispMode = ispOptions[ispOptionIdx];
//we get the mpm cand list
const int numMPMs = NUM_MOST_PROBABLE_MODES;
unsigned  uiPreds[numMPMs];

pu.multiRefIdx = 0;

PU::getIntraMPMs( pu, uiPreds );

//we copy only the ISP MPMs
listPointerCopy = &( cu.ispMode == HOR_INTRA_SUBPARTITIONS ? uiRdModeListCopyHor : uiRdModeListCopyVer );
listPointer     = &( cu.ispMode == HOR_INTRA_SUBPARTITIONS ? m_rdModeListWithoutMrlHor : m_rdModeListWithoutMrlVer );
for( int k = 0; k < listPointerCopy->size(); k++ )
{
for( int q = 0; q < numMPMs; q++ )
{
if (listPointerCopy->at(k) == ModeInfo( false, 0, cu.ispMode, uiPreds[q] ))
{
listPointer->push_back( listPointerCopy->at( k ) );
break;
}
}
}
}
cu.ispMode = NOT_INTRA_SUBPARTITIONS;
}
#endif

CHECK( numModesForFullRD != uiRdModeList.size(), "Inconsistent state!" );

// after this point, don't use numModesForFullRD

// PBINTRA fast
if( m_pcEncCfg->getUsePbIntraFast() && !cs.slice->isIntra() && uiRdModeList.size() < numModesAvailable && !cs.slice->getDisableSATDForRD() && ( mtsUsageFlag != 2 || lfnstIdx > 0 ) )
{
double pbintraRatio = (lfnstIdx > 0) ? 1.25 : PBINTRA_RATIO;
int maxSize = -1;
#if JVET_O0925_MIP_SIMPLIFICATIONS
ModeInfo bestMipMode;
int bestMipIdx = -1;
for( int idx = 0; idx < uiRdModeList.size(); idx++ )
{
if( uiRdModeList[idx].mipFlg )
{
bestMipMode = uiRdModeList[idx];
bestMipIdx = idx;
break;
}
}
const int numHadCand = 3;
#else
const int numHadCand = (testMip ? 2 : 1) * 3;
#endif
for (int k = numHadCand - 1; k >= 0; k--)
{
if (CandHadList.size() < (k + 1) || CandHadList[k] > cs.interHad * pbintraRatio) { maxSize = k; }
}
if (maxSize > 0)
{
uiRdModeList.resize(std::min<size_t>(uiRdModeList.size(), maxSize));
#if JVET_O0925_MIP_SIMPLIFICATIONS
if( bestMipIdx >= 0 )
{
if( uiRdModeList.size() <= bestMipIdx )
{
uiRdModeList.push_back(bestMipMode);
}
}
#endif
if ( testISP )
{
#if JVET_O0502_ISP_CLEANUP
m_ispCandListHor.resize(std::min<size_t>(m_ispCandListHor.size(), maxSize));
#else
m_rdModeListWithoutMrlHor.resize(std::min<size_t>(m_rdModeListWithoutMrlHor.size(), maxSize));
m_rdModeListWithoutMrlVer.resize(std::min<size_t>(m_rdModeListWithoutMrlVer.size(), maxSize));
#endif
}
}
if (maxSize == 0)
{
cs.dist = std::numeric_limits<Distortion>::max();
cs.interHad = 0;

//===== reset context models =====
m_CABACEstimator->getCtx() = SubCtx(Ctx::MipFlag, ctxStartMipFlag);
#if !JVET_O0925_MIP_SIMPLIFICATIONS
m_CABACEstimator->getCtx() = SubCtx(Ctx::MipMode, ctxStartMipMode);
#endif
m_CABACEstimator->getCtx() = SubCtx(Ctx::ISPMode, ctxStartIspMode);
m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaPlanarFlag, ctxStartPlanarFlag);
m_CABACEstimator->getCtx() = SubCtx(Ctx::IntraLumaMpmFlag, ctxStartIntraMode);
m_CABACEstimator->getCtx() = SubCtx(Ctx::MultiRefLineIdx, ctxStartMrlIdx);

return false;
}
}

#if JVET_O0502_ISP_CLEANUP
int numNonISPModes = (int)uiRdModeList.size();//非ISP模式的个数
#endif

if ( testISP )
{
#if JVET_O0502_ISP_CLEANUP
// we reserve positions for ISP in the common full RD list  为ISP模式保留位置
const int maxNumRDModesISP = 16;
for (int i = 0; i < maxNumRDModesISP; i++)
uiRdModeList.push_back(ModeInfo(false, 0, INTRA_SUBPARTITIONS_RESERVED, 0));
#else
//we create a single full RD list that includes all intra modes using regular intra, MRL and ISP
auto* firstIspList  = ispOptions[1] == HOR_INTRA_SUBPARTITIONS ? &m_rdModeListWithoutMrlHor : &m_rdModeListWithoutMrlVer;
auto* secondIspList = ispOptions[1] == HOR_INTRA_SUBPARTITIONS ? &m_rdModeListWithoutMrlVer : &m_rdModeListWithoutMrlHor;

if( !sps.getUseLFNST() && m_pcEncCfg->getUseFastISP() )
{
CHECKD( uiRdModeList.size() > CandCostList.size(), "Error: CandCostList size" );
// find the first non-MRL, non-MIP mode
int indexFirstMode = int(uiRdModeList.size()) - 1; // default is last mode
for (int k = 0; k < int(uiRdModeList.size()); k++)
{
if (uiRdModeList[k].mRefId == 0 && uiRdModeList[k].mipFlg == false)
{
indexFirstMode = k;
break;
}
}
// move the mode indicated by indexFirstMode to the beginning
for (int idx = indexFirstMode - 1; idx >= 0; idx--)
{
std::swap(uiRdModeList[idx], uiRdModeList[idx + 1]);
std::swap(CandCostList[idx], CandCostList[idx + 1]);
}
//insert all ISP modes after the first non-mrl mode
uiRdModeList.insert(uiRdModeList.begin() + 1, secondIspList->begin(), secondIspList->end());
uiRdModeList.insert(uiRdModeList.begin() + 1, firstIspList->begin(), firstIspList->end());
}
else
{
//insert all ISP modes at the end of the current list
uiRdModeList.insert( uiRdModeList.end(), secondIspList->begin(), secondIspList->end() );
uiRdModeList.insert( uiRdModeList.end(), firstIspList->begin() , firstIspList->end()  );
}
#endif
}

//===== check modes (using r-d costs) =====
ModeInfo       uiBestPUMode;//最优PU模式
int            bestBDPCMMode = 0;
double         bestCostNonBDPCM = MAX_DOUBLE;

CodingStructure *csTemp = m_pTempCS[gp_sizeIdxInfo->idxFrom( cu.lwidth() )][gp_sizeIdxInfo->idxFrom( cu.lheight() )];
CodingStructure *csBest = m_pBestCS[gp_sizeIdxInfo->idxFrom( cu.lwidth() )][gp_sizeIdxInfo->idxFrom( cu.lheight() )];

csTemp->slice = cs.slice;
csBest->slice = cs.slice;
csTemp->initStructData();
csBest->initStructData();
#if JVET_O0050_LOCAL_DUAL_TREE
csTemp->picture = cs.picture;
csBest->picture = cs.picture;
#endif

#if !JVET_O0925_MIP_SIMPLIFICATIONS
m_bestCostNonMip = MAX_DOUBLE;
#endif
static_vector<int, FAST_UDI_MAX_RDMODE_NUM> rdModeIdxList;//记录RD索引的列表
if (testMip)
{
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> rdModeListTemp;//临时RD列表
for( int i = 0; i < uiRdModeList.size(); i++)
{
if( !uiRdModeList[i].mipFlg && uiRdModeList[i].ispMod==NOT_INTRA_SUBPARTITIONS )//非MIP模式非ISP模式排在列表的最前面
{
rdModeListTemp.push_back( uiRdModeList[i] );
rdModeIdxList.push_back( i );
}
}
for( int i = 0; i < uiRdModeList.size(); i++)
{
if( uiRdModeList[i].mipFlg || uiRdModeList[i].ispMod!=NOT_INTRA_SUBPARTITIONS )//MIP模式或者ISP模式排在后面
{
rdModeListTemp.push_back( uiRdModeList[i] );
rdModeIdxList.push_back( i );
}
}
#if JVET_O0925_MIP_SIMPLIFICATIONS
uiRdModeList.resize(rdModeListTemp.size());
#endif
for( int i = 0; i < uiRdModeList.size(); i++)
{
uiRdModeList[i] = rdModeListTemp[i];//重排后的列表赋值给原列表
}
}
#if JVET_O0925_MIP_SIMPLIFICATIONS
else
{
static_vector<ModeInfo, FAST_UDI_MAX_RDMODE_NUM> rdModeListTemp;
for( int i = 0; i < uiRdModeList.size(); i++ )
{
if( !uiRdModeList[i].mipFlg  )
{
rdModeListTemp.push_back( uiRdModeList[i] );
}
}
uiRdModeList.resize(rdModeListTemp.size());
for( int i = 0; i < rdModeListTemp.size(); i++ )
{
uiRdModeList[i] = rdModeListTemp[i];
}
}
#endif

// just to be sure
numModesForFullRD = ( int ) uiRdModeList.size();
#if !JVET_O0502_ISP_CLEANUP
PartSplit intraSubPartitionsProcOrder = TU_NO_ISP;
int       bestNormalIntraModeIndex    = -1;
#endif
TUIntraSubPartitioner subTuPartitioner( partitioner );
if( !cu.ispMode && !cu.mtsFlag )
{
m_modeCtrl->setMtsFirstPassNoIspCost( MAX_DOUBLE );
}
#if !JVET_O0502_ISP_CLEANUP
bool      ispHorAllZeroCbfs = false, ispVerAllZeroCbfs = false;
#endif

for (int mode = -2 * int(testBDPCM); mode < (int)uiRdModeList.size(); mode++)//遍历列表每个模式进行完整的RD代价计算选出最优的模式
{
// set CU/PU to luma prediction mode
ModeInfo uiOrgMode;
if ( mode < 0 )//BDPCM
{
cu.bdpcmMode = -mode;

#if JVET_O0315_RDPCM_INTRAMODE_ALIGN
uiOrgMode = ModeInfo(false, 0, NOT_INTRA_SUBPARTITIONS, cu.bdpcmMode == 2 ? VER_IDX : HOR_IDX);
#else
unsigned mpm_pred[NUM_MOST_PROBABLE_MODES];
PU::getIntraMPMs(pu, mpm_pred);
uiOrgMode = ModeInfo(false, 0, NOT_INTRA_SUBPARTITIONS, mpm_pred[0]);
#endif
cu.mipFlag                     = uiOrgMode.mipFlg;//更新CU和PU的信息
cu.ispMode                     = uiOrgMode.ispMod;
pu.multiRefIdx                 = uiOrgMode.mRefId;
pu.intraDir[CHANNEL_TYPE_LUMA] = uiOrgMode.modeId;
}
else
{
cu.bdpcmMode = 0;
#if JVET_O0502_ISP_CLEANUP
if (uiRdModeList[mode].ispMod == INTRA_SUBPARTITIONS_RESERVED)//如果是ISP模式
{
if (mode == numNonISPModes) // the list needs to be sorted only once ISP列表仅重排一次
{
xSortISPCandList(bestCurrentCost, csBest->cost);//重新排列ISP列表，水平和垂直列表中的模式一样
}
xGetNextISPMode(uiRdModeList[mode], (mode > 0 ? &uiRdModeList[mode - 1] : nullptr), Size(width, height));//获取下一个ISP模式
if (uiRdModeList[mode].ispMod == INTRA_SUBPARTITIONS_RESERVED)
continue;
}
#endif
uiOrgMode = uiRdModeList[mode];
cu.mipFlag                     = uiOrgMode.mipFlg;
cu.ispMode                     = uiOrgMode.ispMod;
pu.multiRefIdx                 = uiOrgMode.mRefId;
pu.intraDir[CHANNEL_TYPE_LUMA] = uiOrgMode.modeId;

CHECK(cu.mipFlag && pu.multiRefIdx, "Error: combination of MIP and MRL not supported");
CHECK(pu.multiRefIdx && (pu.intraDir[0] == PLANAR_IDX), "Error: combination of MRL and Planar mode not supported");
CHECK(cu.ispMode && cu.mipFlag, "Error: combination of ISP and MIP not supported");
CHECK(cu.ispMode && pu.multiRefIdx, "Error: combination of ISP and MRL not supported");
#if !JVET_O0502_ISP_CLEANUP
if( cu.ispMode )
{
intraSubPartitionsProcOrder = CU::getISPType( cu, COMPONENT_Y );
bool tuIsDividedInRows = CU::divideTuInRows( cu );
if ( ( tuIsDividedInRows && ispHorAllZeroCbfs ) || ( !tuIsDividedInRows && ispVerAllZeroCbfs ) )
{
continue;
}
if( m_intraModeDiagRatio.at( bestNormalIntraModeIndex ) > 1.25 )
{
continue;
}
if( ( m_intraModeHorVerRatio.at( bestNormalIntraModeIndex ) > 1.25 && tuIsDividedInRows ) || ( m_intraModeHorVerRatio.at( bestNormalIntraModeIndex ) < 0.8 && !tuIsDividedInRows ) )
{
continue;
}
}
#endif
}

// set context models
m_CABACEstimator->getCtx() = ctxStart;

// determine residual for partition
cs.initSubStructure( *csTemp, partitioner.chType, cs.area, true );

bool tmpValidReturn = false;
if( cu.ispMode )//如果是ISP模式
{
#if JVET_O0502_ISP_CLEANUP
tmpValidReturn = xIntraCodingLumaISP(*csTemp, subTuPartitioner, bestCurrentCost);//进行ISP模式的残差变换，量化，计算律师真代价等操作
if (csTemp->tus.size() == 0)
{
// no TUs were coded
csTemp->cost = MAX_DOUBLE;
continue;
}
if (!cu.mtsFlag && !cu.lfnstIdx)
{
// we save the data for future tests
m_ispTestedModes.setModeResults((ISPType)cu.ispMode, (int)uiOrgMode.modeId, (int)csTemp->tus.size(), csTemp->cus[0]->firstTU->cbf[COMPONENT_Y] ? csTemp->cost : MAX_DOUBLE, csBest->cost);
}
#else
tmpValidReturn = xRecurIntraCodingLumaQT( *csTemp, subTuPartitioner, bestCurrentCost, 0, intraSubPartitionsProcOrder, false,
mtsCheckRangeFlag, mtsFirstCheckId, mtsLastCheckId, moreProbMTSIdxFirst );
#endif
}
else//否则进行普通的角度预测残差计算
{
#if !JVET_O0925_MIP_SIMPLIFICATIONS
if( ! fastMip )
{
m_bestCostNonMip = MAX_DOUBLE;
}
#endif
tmpValidReturn = xRecurIntraCodingLumaQT( *csTemp, partitioner, uiBestPUMode.ispMod ? bestCurrentCost : MAX_DOUBLE, -1, TU_NO_ISP, uiBestPUMode.ispMod,
mtsCheckRangeFlag, mtsFirstCheckId, mtsLastCheckId, moreProbMTSIdxFirst );
}

#if JVET_O0502_ISP_CLEANUP
if (!cu.ispMode && !cu.mtsFlag && !cu.lfnstIdx && !cu.bdpcmMode && !pu.multiRefIdx && !cu.mipFlag && testISP)
{
m_regIntraRDListWithCosts.push_back(ModeInfoWithCost(cu.mipFlag, pu.multiRefIdx, cu.ispMode, uiOrgMode.modeId, csTemp->cost));
}
#endif

if( cu.ispMode && !csTemp->cus[0]->firstTU->cbf[COMPONENT_Y] )
{
#if !JVET_O0502_ISP_CLEANUP
if( !sps.getUseLFNST() )
{
if ( cu.ispMode == HOR_INTRA_SUBPARTITIONS )
{
ispHorAllZeroCbfs |= ( m_pcEncCfg->getUseFastISP() && csTemp->tus[0]->lheight() > 2 && csTemp->cost >= bestCurrentCost );
}
else
{
ispVerAllZeroCbfs |= ( m_pcEncCfg->getUseFastISP() && csTemp->tus[0]->lwidth() > 2 && csTemp->cost >= bestCurrentCost );
}
}
#endif
csTemp->cost = MAX_DOUBLE;
csTemp->costDbOffset = 0;
tmpValidReturn = false;
}
validReturn |= tmpValidReturn;

if( sps.getUseLFNST() && mtsUsageFlag == 1 && !cu.ispMode && mode >= 0 )
{
m_modeCostStore[ lfnstIdx ][ testMip ? rdModeIdxList[ mode ] : mode ] = tmpValidReturn ? csTemp->cost : ( MAX_DOUBLE / 2.0 ); //(MAX_DOUBLE / 2.0) ??
}

#if JVET_O0502_ISP_CLEANUP
DTRACE(g_trace_ctx, D_INTRA_COST, "IntraCost T [x=%d,y=%d,w=%d,h=%d] %f (%d,%d,%d,%d,%d,%d) \n", cu.blocks[0].x,
cu.blocks[0].y, (int)width, (int)height, csTemp->cost, uiOrgMode.modeId, uiOrgMode.ispMod,
pu.multiRefIdx, cu.mipFlag, cu.lfnstIdx, cu.mtsFlag);
#else
DTRACE( g_trace_ctx, D_INTRA_COST, "IntraCost T %f (%d) \n", csTemp->cost, uiOrgMode.modeId );
#endif

if( tmpValidReturn )//如果是有效返回值，比较最优模式代价和当前模式代价
{
// check r-d cost
if( csTemp->cost < csBest->cost )
{
std::swap( csTemp, csBest );

uiBestPUMode  = uiOrgMode;
bestBDPCMMode = cu.bdpcmMode;
if( sps.getUseLFNST() && mtsUsageFlag == 1 && !cu.ispMode )
{
m_bestModeCostStore[ lfnstIdx ] = csBest->cost; //cs.cost;
m_bestModeCostValid[ lfnstIdx ] = true;
}
if( csBest->cost < bestCurrentCost )
{
bestCurrentCost = csBest->cost;
}
if( !cu.ispMode && !cu.mtsFlag )
{
m_modeCtrl->setMtsFirstPassNoIspCost( csBest->cost );
}
}
if( !cu.ispMode && !cu.bdpcmMode && csBest->cost < bestCostNonBDPCM )
{
bestCostNonBDPCM = csBest->cost;
#if !JVET_O0502_ISP_CLEANUP
bestNormalIntraModeIndex = mode;
#endif
}
}

csTemp->releaseIntermediateData();//清空CU,PU,TU
#if JVET_O0050_LOCAL_DUAL_TREE
if( m_pcEncCfg->getFastLocalDualTreeMode() )
{
if( cu.isConsIntra() && !cu.slice->isIntra() && csBest->cost != MAX_DOUBLE && costInterCU != COST_UNKNOWN && mode >= 0 )
{
if( m_pcEncCfg->getFastLocalDualTreeMode() == 2 )
{
//Note: only try one intra mode, which is especially useful to reduce EncT for LDB case (around 4%)
break;
}
else
{
if( csBest->cost > costInterCU * 1.5 )
{
break;
}
}
}
}
#endif
} // Mode loop 循环结束，选出最优的模式
cu.ispMode = uiBestPUMode.ispMod;//最优的ISP模式

if( validReturn )
{
cs.useSubStructure( *csBest, partitioner.chType, pu.singleChan( CHANNEL_TYPE_LUMA ), true, true, keepResi, keepResi );
}
csBest->releaseIntermediateData();
if( validReturn )
{
//=== update PU data ====
cu.mipFlag = uiBestPUMode.mipFlg;
pu.multiRefIdx = uiBestPUMode.mRefId;
pu.intraDir[ CHANNEL_TYPE_LUMA ] = uiBestPUMode.modeId;
cu.bdpcmMode = bestBDPCMMode;
}
}

//===== reset context models =====
m_CABACEstimator->getCtx() = ctxStart;

return validReturn;
}

• 点赞
• 收藏
• 分享
• 文章举报