taosdata / TDengine / #3565

coverage: 51.098% (-11.1%) from 62.21%
#3565

push

travis-ci

web-flow 
Merge pull request #29316 from taosdata/enh/3.0/TD-33266

enh(ut):Add wal & config UT.

111558 of 284773 branches covered (39.17%)

Branch coverage included in aggregate %.

1 of 2 new or added lines in 2 files covered. (50.0%)

39015 existing lines in 102 files now uncovered.

177882 of 281666 relevant lines covered (63.15%)

15090998.35 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/source/dnode/vnode/src/tsdb/tsdbMergeTree.c
1 
/*
2 
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3 
 *
4 
 * This program is free software: you can use, redistribute, and/or modify
5 
 * it under the terms of the GNU Affero General Public License, version 3
6 
 * or later ("AGPL"), as published by the Free Software Foundation.
7 
 *
8 
 * This program is distributed in the hope that it will be useful, but WITHOUT
9 
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 
 * FITNESS FOR A PARTICULAR PURPOSE.
11 
 *
12 
 * You should have received a copy of the GNU Affero General Public License
13 
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
14 
 */
15 









16



#include "tsdb.h"










17



#include "tsdbFSet2.h"










18



#include "tsdbMerge.h"










19



#include "tsdbReadUtil.h"










20



#include "tsdbSttFileRW.h"










21



#include "tsdbUtil2.h"










22












23



static void tLDataIterClose2(SLDataIter *pIter);










24












25



// SLDataIter =================================================











UNCOV

26



int32_t tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, SSttBlockLoadInfo **pInfo) {




×








UNCOV

27



  *pInfo = NULL;




×







28













UNCOV

29



  SSttBlockLoadInfo *pLoadInfo = taosMemoryCalloc(1, sizeof(SSttBlockLoadInfo));





×








UNCOV

30



  if (pLoadInfo == NULL) {





×







31



    return terrno;




×







32



  }










33













UNCOV

34



  pLoadInfo->blockData[0].sttBlockIndex = -1;




×








UNCOV

35



  pLoadInfo->blockData[1].sttBlockIndex = -1;




×







36













UNCOV

37



  pLoadInfo->currentLoadBlockIndex = 1;




×







38













UNCOV

39



  int32_t code = tBlockDataCreate(&pLoadInfo->blockData[0].data);




×








UNCOV

40



  if (code) {





×







41



    taosMemoryFreeClear(pLoadInfo);





×







42



    return code;




×







43



  }










44













UNCOV

45



  code = tBlockDataCreate(&pLoadInfo->blockData[1].data);




×








UNCOV

46



  if (code) {





×







47



    taosMemoryFreeClear(pLoadInfo);





×







48



    return code;




×







49



  }










50













UNCOV

51



  pLoadInfo->aSttBlk = taosArrayInit(4, sizeof(SSttBlk));




×








UNCOV

52



  if (pLoadInfo->aSttBlk == NULL) {





×








UNCOV

53



    taosMemoryFreeClear(pLoadInfo);





×








UNCOV

54



    return terrno;




×







55



  }










56













UNCOV

57



  pLoadInfo->pSchema = pSchema;




×








UNCOV

58



  pLoadInfo->colIds = colList;




×








UNCOV

59



  pLoadInfo->numOfCols = numOfCols;




×







60













UNCOV

61



  *pInfo = pLoadInfo;




×








UNCOV

62



  return code;




×







63



}










64













UNCOV

65



static void freeItem(void *pValue) {




×








UNCOV

66



  SValue *p = (SValue *)pValue;




×








UNCOV

67



  if (IS_VAR_DATA_TYPE(p->type)) {





×







68



    taosMemoryFree(p->pData);





×







69



  }











UNCOV

70



}




×







71













UNCOV

72



void destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) {




×








UNCOV

73



  if (pLoadInfo == NULL) {





×







74



    return;




×







75



  }










76













UNCOV

77



  pLoadInfo->currentLoadBlockIndex = 1;




×







78













UNCOV

79



  SBlockDataInfo *pInfo = &pLoadInfo->blockData[0];




×








UNCOV

80



  tBlockDataDestroy(&pInfo->data);




×








UNCOV

81



  pInfo->sttBlockIndex = -1;




×








UNCOV

82



  pInfo->pin = false;




×







83













UNCOV

84



  pInfo = &pLoadInfo->blockData[1];




×








UNCOV

85



  tBlockDataDestroy(&pInfo->data);




×








UNCOV

86



  pInfo->sttBlockIndex = -1;




×








UNCOV

87



  pInfo->pin = false;




×







88













UNCOV

89



  taosArrayDestroy(pLoadInfo->info.pUid);




×








UNCOV

90



  taosArrayDestroyEx(pLoadInfo->info.pFirstKey, freeItem);




×








UNCOV

91



  taosArrayDestroyEx(pLoadInfo->info.pLastKey, freeItem);




×








UNCOV

92



  taosArrayDestroy(pLoadInfo->info.pCount);




×








UNCOV

93



  taosArrayDestroy(pLoadInfo->info.pFirstTs);




×








UNCOV

94



  taosArrayDestroy(pLoadInfo->info.pLastTs);




×







95













UNCOV

96



  pLoadInfo->info.pUid = NULL;




×








UNCOV

97



  pLoadInfo->info.pFirstKey = NULL;




×








UNCOV

98



  pLoadInfo->info.pLastKey = NULL;




×








UNCOV

99



  pLoadInfo->info.pCount = NULL;




×








UNCOV

100



  pLoadInfo->info.pFirstTs = NULL;




×








UNCOV

101



  pLoadInfo->info.pLastTs = NULL;




×







102













UNCOV

103



  taosArrayDestroy(pLoadInfo->aSttBlk);




×








UNCOV

104



  taosMemoryFree(pLoadInfo);





×







105



}










106













UNCOV

107



void destroyLDataIter(SLDataIter *pIter) {




×








UNCOV

108



  tLDataIterClose2(pIter);




×








UNCOV

109



  destroySttBlockLoadInfo(pIter->pBlockLoadInfo);




×








UNCOV

110



  taosMemoryFree(pIter);





×








UNCOV

111



}




×







112













UNCOV

113



void destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost) {




×








UNCOV

114



  if (pLDataIterArray == NULL) {





×








UNCOV

115



    return;




×







116



  }










117













UNCOV

118



  int32_t numOfLevel = taosArrayGetSize(pLDataIterArray);




×








UNCOV

119



  for (int32_t i = 0; i < numOfLevel; ++i) {





×








UNCOV

120



    SArray *pList = taosArrayGetP(pLDataIterArray, i);




×








UNCOV

121



    for (int32_t j = 0; j < taosArrayGetSize(pList); ++j) {





×








UNCOV

122



      SLDataIter *pIter = taosArrayGetP(pList, j);




×








UNCOV

123



      if (pIter->pBlockLoadInfo != NULL) {





×








UNCOV

124



        SSttBlockLoadCostInfo *pCost = &pIter->pBlockLoadInfo->cost;




×








UNCOV

125



        if (pLoadCost != NULL) {





×








UNCOV

126



          pLoadCost->loadBlocks += pCost->loadBlocks;




×








UNCOV

127



          pLoadCost->loadStatisBlocks += pCost->loadStatisBlocks;




×








UNCOV

128



          pLoadCost->blockElapsedTime += pCost->blockElapsedTime;




×








UNCOV

129



          pLoadCost->statisElapsedTime += pCost->statisElapsedTime;




×







130



        }










131



      }










132













UNCOV

133



      destroyLDataIter(pIter);




×







134



    }










135













UNCOV

136



    taosArrayDestroy(pList);




×







137



  }










138













UNCOV

139



  taosArrayDestroy(pLDataIterArray);




×







140



}










141












142



// choose the unpinned slot to load next data block











UNCOV

143



static void updateBlockLoadSlot(SSttBlockLoadInfo *pLoadInfo) {




×








UNCOV

144



  int32_t nextSlotIndex = pLoadInfo->currentLoadBlockIndex ^ 1;




×








UNCOV

145



  if (pLoadInfo->blockData[nextSlotIndex].pin) {





×







146



    nextSlotIndex = nextSlotIndex ^ 1;




×







147



  }










148













UNCOV

149



  pLoadInfo->currentLoadBlockIndex = nextSlotIndex;




×








UNCOV

150



}




×







151













UNCOV

152



static int32_t loadLastBlock(SLDataIter *pIter, const char *idStr, SBlockData **pResBlock) {




×








UNCOV

153



  if (pResBlock != NULL) {





×








UNCOV

154



    *pResBlock = NULL;




×







155



  }










156













UNCOV

157



  int32_t            code = 0;




×








UNCOV

158



  SSttBlockLoadInfo *pInfo = pIter->pBlockLoadInfo;




×







159













UNCOV

160



  if (pInfo->blockData[0].sttBlockIndex == pIter->iSttBlk) {





×








UNCOV

161



    if (pInfo->currentLoadBlockIndex != 0) {





×








UNCOV

162



      tsdbDebug("current load index is set to 0, block index:%d, fileVer:%" PRId64 ", due to uid:%" PRIu64





×







163



                ", load data, %s",










164



                pIter->iSttBlk, pIter->cid, pIter->uid, idStr);











UNCOV

165



      pInfo->currentLoadBlockIndex = 0;




×







166



    }










167













UNCOV

168



    *pResBlock = &pInfo->blockData[0].data;




×








UNCOV

169



    return code;




×







170



  }










171













UNCOV

172



  if (pInfo->blockData[1].sttBlockIndex == pIter->iSttBlk) {





×








UNCOV

173



    if (pInfo->currentLoadBlockIndex != 1) {





×








UNCOV

174



      tsdbDebug("current load index is set to 1, block index:%d, fileVer:%" PRId64 ", due to uid:%" PRIu64





×







175



                ", load data, %s",










176



                pIter->iSttBlk, pIter->cid, pIter->uid, idStr);











UNCOV

177



      pInfo->currentLoadBlockIndex = 1;




×







178



    }










179













UNCOV

180



    *pResBlock = &pInfo->blockData[1].data;




×








UNCOV

181



    return code;




×







182



  }










183












184



  if (pIter->pSttBlk == NULL || pInfo->pSchema == NULL) {





×







185



    return code;




×







186



  }










187













UNCOV

188



  updateBlockLoadSlot(pInfo);




×








UNCOV

189



  int64_t st = taosGetTimestampUs();




×







190













UNCOV

191



  SBlockData *pBlock = &pInfo->blockData[pInfo->currentLoadBlockIndex].data;




×








UNCOV

192



  code = tsdbSttFileReadBlockDataByColumn(pIter->pReader, pIter->pSttBlk, pBlock, pInfo->pSchema, &pInfo->colIds[1],




×








UNCOV

193



                                          pInfo->numOfCols - 1);




×








UNCOV

194



  if (code != TSDB_CODE_SUCCESS) {





×







195



    return code;




×







196



  }










197













UNCOV

198



  double el = (taosGetTimestampUs() - st) / 1000.0;




×








UNCOV

199



  pInfo->cost.blockElapsedTime += el;




×








UNCOV

200



  pInfo->cost.loadBlocks += 1;




×







201













UNCOV

202



  tsdbDebug("read stt block, total load:%" PRId64 ", trigger by uid:%" PRIu64 ", stt-fileVer:%" PRId64





×







203



            ", last block index:%d, entry:%d, rows:%d, uidRange:%" PRId64 "-%" PRId64 " tsRange:%" PRId64 "-%" PRId64










204



            " %p, elapsed time:%.2f ms, %s",










205



            pInfo->cost.loadBlocks, pIter->uid, pIter->cid, pIter->iSttBlk, pInfo->currentLoadBlockIndex, pBlock->nRow,










206



            pIter->pSttBlk->minUid, pIter->pSttBlk->maxUid, pIter->pSttBlk->minKey, pIter->pSttBlk->maxKey, pBlock, el,










207



            idStr);










208













UNCOV

209



  pInfo->blockData[pInfo->currentLoadBlockIndex].sttBlockIndex = pIter->iSttBlk;




×








UNCOV

210



  pIter->iRow = (pIter->backward) ? pInfo->blockData[pInfo->currentLoadBlockIndex].data.nRow : -1;





×







211













UNCOV

212



  tsdbDebug("last block index list:%d, %d, rowIndex:%d %s", pInfo->blockData[0].sttBlockIndex,





×







213



            pInfo->blockData[1].sttBlockIndex, pIter->iRow, idStr);










214













UNCOV

215



  *pResBlock = &pInfo->blockData[pInfo->currentLoadBlockIndex].data;




×








UNCOV

216



  return code;




×







217



}










218












219



// find the earliest block that contains the required records










220



static FORCE_INLINE int32_t findEarliestIndex(int32_t index, uint64_t uid, const SSttBlk *pBlockList, int32_t num,










221



                                              int32_t backward) {











UNCOV

222



  int32_t i = index;




×








UNCOV

223



  int32_t step = backward ? 1 : -1;





×








UNCOV

224



  while (i >= 0 && i < num && uid >= pBlockList[i].minUid && uid <= pBlockList[i].maxUid) {





×








UNCOV

225



    i += step;




×







226



  }











UNCOV

227



  return i - step;




×







228



}










229













UNCOV

230



static int32_t binarySearchForStartBlock(SSttBlk *pBlockList, int32_t num, uint64_t uid, int32_t backward) {




×








UNCOV

231



  int32_t midPos = -1;




×








UNCOV

232



  if (num <= 0) {





×








UNCOV

233



    return -1;




×







234



  }










235













UNCOV

236



  int32_t firstPos = 0;




×








UNCOV

237



  int32_t lastPos = num - 1;




×







238












239



  // find the first position which is bigger than the key











UNCOV

240



  if ((uid > pBlockList[lastPos].maxUid) || (uid < pBlockList[firstPos].minUid)) {





×








UNCOV

241



    return -1;




×







242



  }










243













UNCOV

244



  while (1) {




×








UNCOV

245



    if (uid >= pBlockList[firstPos].minUid && uid <= pBlockList[firstPos].maxUid) {





×








UNCOV

246



      return findEarliestIndex(firstPos, uid, pBlockList, num, backward);




×







247



    }










248













UNCOV

249



    if (uid > pBlockList[lastPos].maxUid || uid < pBlockList[firstPos].minUid) {





×








UNCOV

250



      return -1;




×







251



    }










252













UNCOV

253



    int32_t numOfRows = lastPos - firstPos + 1;




×








UNCOV

254



    midPos = (numOfRows >> 1u) + firstPos;




×







255













UNCOV

256



    if (uid < pBlockList[midPos].minUid) {





×








UNCOV

257



      lastPos = midPos - 1;




×








UNCOV

258



    } else if (uid > pBlockList[midPos].maxUid) {





×








UNCOV

259



      firstPos = midPos + 1;




×







260



    } else {











UNCOV

261



      return findEarliestIndex(midPos, uid, pBlockList, num, backward);




×







262



    }










263



  }










264



}










265












266



static FORCE_INLINE int32_t findEarliestRow(int32_t index, uint64_t uid, const uint64_t *uidList, int32_t num,










267



                                            int32_t backward) {











UNCOV

268



  int32_t i = index;




×








UNCOV

269



  int32_t step = backward ? 1 : -1;





×








UNCOV

270



  while (i >= 0 && i < num && uid == uidList[i]) {





×








UNCOV

271



    i += step;




×







272



  }











UNCOV

273



  return i - step;




×







274



}










275













UNCOV

276



static int32_t binarySearchForStartRowIndex(uint64_t *uidList, int32_t num, uint64_t uid, int32_t backward) {




×








UNCOV

277



  int32_t firstPos = 0;




×








UNCOV

278



  int32_t lastPos = num - 1;




×







279












280



  // find the first position which is bigger than the key











UNCOV

281



  if ((uid > uidList[lastPos]) || (uid < uidList[firstPos])) {





×







282



    return -1;




×







283



  }










284













UNCOV

285



  while (1) {




×








UNCOV

286



    if (uid == uidList[firstPos]) {





×








UNCOV

287



      return findEarliestRow(firstPos, uid, uidList, num, backward);




×







288



    }










289













UNCOV

290



    if (uid > uidList[lastPos] || uid < uidList[firstPos]) {





×








UNCOV

291



      return -1;




×







292



    }










293













UNCOV

294



    int32_t numOfRows = lastPos - firstPos + 1;




×








UNCOV

295



    int32_t midPos = (numOfRows >> 1u) + firstPos;




×







296













UNCOV

297



    if (uid < uidList[midPos]) {





×








UNCOV

298



      lastPos = midPos - 1;




×








UNCOV

299



    } else if (uid > uidList[midPos]) {





×








UNCOV

300



      firstPos = midPos + 1;




×







301



    } else {











UNCOV

302



      return findEarliestRow(midPos, uid, uidList, num, backward);




×







303



    }










304



  }










305



}










306













UNCOV

307



static int32_t extractSttBlockInfo(SLDataIter *pIter, const TSttBlkArray *pArray, SSttBlockLoadInfo *pBlockLoadInfo,




×







308



                                   uint64_t suid) {











UNCOV

309



  void   *px = NULL;




×








UNCOV

310



  int32_t code = TSDB_CODE_SUCCESS;




×








UNCOV

311



  if (TARRAY2_SIZE(pArray) <= 0) {





×








UNCOV

312



    return code;




×







313



  }










314













UNCOV

315



  SSttBlk *pStart = &pArray->data[0];




×








UNCOV

316



  SSttBlk *pEnd = &pArray->data[TARRAY2_SIZE(pArray) - 1];




×







317












318



  // all identical











UNCOV

319



  if (pStart->suid == pEnd->suid) {





×








UNCOV

320



    if (pStart->suid != suid) {  // no qualified stt block existed





×








UNCOV

321



      taosArrayClear(pBlockLoadInfo->aSttBlk);




×








UNCOV

322



      pIter->iSttBlk = -1;




×








UNCOV

323



      return TSDB_CODE_SUCCESS;




×







324



    } else {  // all blocks are qualified











UNCOV

325



      taosArrayClear(pBlockLoadInfo->aSttBlk);




×








UNCOV

326



      px = taosArrayAddBatch(pBlockLoadInfo->aSttBlk, pArray->data, pArray->size);




×








UNCOV

327



      if (px == NULL) {





×







328



        return terrno;




×







329



      }










330



    }










331



  } else {











UNCOV

332



    SArray *pTmp = taosArrayInit(TARRAY2_SIZE(pArray), sizeof(SSttBlk));




×








UNCOV

333



    if (pTmp == NULL) {





×







334



      return terrno;




×







335



    }










336













UNCOV

337



    for (int32_t i = 0; i < TARRAY2_SIZE(pArray); ++i) {





×








UNCOV

338



      SSttBlk *p = &pArray->data[i];




×








UNCOV

339



      if (p->suid < suid) {





×








UNCOV

340



        continue;




×







341



      }










342













UNCOV

343



      if (p->suid == suid) {





×








UNCOV

344



        void *px = taosArrayPush(pTmp, p);




×








UNCOV

345



        if (px == NULL) {





×







346



          code = terrno;




×







347



          break;




×







348



        }











UNCOV

349



      } else if (p->suid > suid) {





×








UNCOV

350



        break;




×







351



      }










352



    }










353













UNCOV

354



    taosArrayDestroy(pBlockLoadInfo->aSttBlk);




×








UNCOV

355



    pBlockLoadInfo->aSttBlk = pTmp;




×







356



  }










357













UNCOV

358



  return code;




×







359



}










360













UNCOV

361



static int32_t tValueDupPayload(SValue *pVal) {




×








UNCOV

362



  if (IS_VAR_DATA_TYPE(pVal->type)) {





×







363



    char *p = (char *)pVal->pData;




×







364



    char *pBuf = taosMemoryMalloc(pVal->nData);





×








UNCOV

365



    if (pBuf == NULL) {





×







366



      return terrno;




×







367



    }










368













UNCOV

369



    memcpy(pBuf, p, pVal->nData);




×








UNCOV

370



    pVal->pData = (uint8_t *)pBuf;




×







371



  }










372













UNCOV

373



  return TSDB_CODE_SUCCESS;




×







374



}










375













UNCOV

376



static int32_t loadSttStatisticsBlockData(SSttFileReader *pSttFileReader, SSttBlockLoadInfo *pBlockLoadInfo,




×







377



                                          TStatisBlkArray *pStatisBlkArray, uint64_t suid, const char *id) {











UNCOV

378



  int32_t code = TSDB_CODE_SUCCESS;




×








UNCOV

379



  int32_t lino = 0;




×








UNCOV

380



  void   *px = NULL;




×








UNCOV

381



  int32_t startIndex = 0;




×








UNCOV

382



  int32_t ret = 0;




×







383













UNCOV

384



  int32_t numOfBlocks = TARRAY2_SIZE(pStatisBlkArray);




×








UNCOV

385



  if (numOfBlocks <= 0) {





×








UNCOV

386



    return code;




×







387



  }










388













UNCOV

389



  while ((startIndex < numOfBlocks) && (pStatisBlkArray->data[startIndex].maxTbid.suid < suid)) {





×








UNCOV

390



    ++startIndex;




×







391



  }










392













UNCOV

393



  if (startIndex >= numOfBlocks || pStatisBlkArray->data[startIndex].minTbid.suid > suid) {





×








UNCOV

394



    return 0;




×







395



  }










396













UNCOV

397



  int32_t endIndex = startIndex;




×








UNCOV

398



  while (endIndex < numOfBlocks && pStatisBlkArray->data[endIndex].minTbid.suid <= suid) {





×








UNCOV

399



    ++endIndex;




×







400



  }










401













UNCOV

402



  int32_t num = endIndex - startIndex;




×








UNCOV

403



  pBlockLoadInfo->cost.loadStatisBlocks += num;




×







404












405



  STbStatisBlock block;











UNCOV

406



  code = tStatisBlockInit(&block);




×








UNCOV

407



  QUERY_CHECK_CODE(code, lino, _end);





×







408













UNCOV

409



  int64_t st = taosGetTimestampUs();




×







410













UNCOV

411



  for (int32_t k = startIndex; k < endIndex; ++k) {





×








UNCOV

412



    code = tsdbSttFileReadStatisBlock(pSttFileReader, &pStatisBlkArray->data[k], &block);




×








UNCOV

413



    QUERY_CHECK_CODE(code, lino, _end);





×







414













UNCOV

415



    int32_t i = 0;




×








UNCOV

416



    int32_t rows = block.numOfRecords;




×








UNCOV

417



    while (i < rows && ((int64_t *)block.suids.data)[i] != suid) {





×








UNCOV

418



      ++i;




×







419



    }










420












421



    // existed











UNCOV

422



    if (i < rows) {





×








UNCOV

423



      SSttTableRowsInfo *pInfo = &pBlockLoadInfo->info;




×







424













UNCOV

425



      if (pInfo->pUid == NULL) {





×








UNCOV

426



        pInfo->pUid = taosArrayInit(rows, sizeof(int64_t));




×








UNCOV

427



        pInfo->pFirstTs = taosArrayInit(rows, sizeof(int64_t));




×








UNCOV

428



        pInfo->pLastTs = taosArrayInit(rows, sizeof(int64_t));




×








UNCOV

429



        pInfo->pCount = taosArrayInit(rows, sizeof(int64_t));




×







430













UNCOV

431



        pInfo->pFirstKey = taosArrayInit(rows, sizeof(SValue));




×








UNCOV

432



        pInfo->pLastKey = taosArrayInit(rows, sizeof(SValue));




×







433













UNCOV

434



        if (pInfo->pUid == NULL || pInfo->pFirstTs == NULL || pInfo->pLastTs == NULL || pInfo->pCount == NULL ||





×








UNCOV

435



            pInfo->pFirstKey == NULL || pInfo->pLastKey == NULL) {





×








UNCOV

436



          code = terrno;




×







437



          goto _end;




×







438



        }










439



      }










440













UNCOV

441



      if (pStatisBlkArray->data[k].maxTbid.suid == suid) {





×








UNCOV

442



        int32_t size = rows - i;




×








UNCOV

443



        int32_t offset = i * sizeof(int64_t);




×







444













UNCOV

445



        px = taosArrayAddBatch(pInfo->pUid, tBufferGetDataAt(&block.uids, offset), size);




×








UNCOV

446



        TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







447













UNCOV

448



        px = taosArrayAddBatch(pInfo->pFirstTs, tBufferGetDataAt(&block.firstKeyTimestamps, offset), size);




×








UNCOV

449



        TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







450













UNCOV

451



        px = taosArrayAddBatch(pInfo->pLastTs, tBufferGetDataAt(&block.lastKeyTimestamps, offset), size);




×








UNCOV

452



        TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







453













UNCOV

454



        px = taosArrayAddBatch(pInfo->pCount, tBufferGetDataAt(&block.counts, offset), size);




×








UNCOV

455



        TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







456













UNCOV

457



        if (block.numOfPKs > 0) {





×








UNCOV

458



          SValue vFirst = {0}, vLast = {0};




×








UNCOV

459



          for (int32_t f = i; f < rows; ++f) {





×








UNCOV

460



            code = tValueColumnGet(&block.firstKeyPKs[0], f, &vFirst);




×








UNCOV

461



            TSDB_CHECK_CODE(code, lino, _end);





×







462













UNCOV

463



            code = tValueDupPayload(&vFirst);




×








UNCOV

464



            TSDB_CHECK_CODE(code, lino, _end);





×







465













UNCOV

466



            px = taosArrayPush(pInfo->pFirstKey, &vFirst);




×








UNCOV

467



            TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







468












469



            // todo add api to clone the original data











UNCOV

470



            code = tValueColumnGet(&block.lastKeyPKs[0], f, &vLast);




×








UNCOV

471



            TSDB_CHECK_CODE(code, lino, _end);





×







472













UNCOV

473



            code = tValueDupPayload(&vLast);




×








UNCOV

474



            TSDB_CHECK_CODE(code, lino, _end);





×







475













UNCOV

476



            px = taosArrayPush(pInfo->pLastKey, &vLast);




×








UNCOV

477



            TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







478



          }










479



        } else {











UNCOV

480



          SValue vFirst = {0};




×








UNCOV

481



          for (int32_t j = 0; j < size; ++j) {





×








UNCOV

482



            px = taosArrayPush(pInfo->pFirstKey, &vFirst);




×








UNCOV

483



            TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







484













UNCOV

485



            px = taosArrayPush(pInfo->pLastKey, &vFirst);




×








UNCOV

486



            TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







487



          }










488



        }










489



      } else {











UNCOV

490



        STbStatisRecord record = {0};




×








UNCOV

491



        while (i < rows) {





×








UNCOV

492



          code = tStatisBlockGet(&block, i, &record);




×








UNCOV

493



          TSDB_CHECK_CODE(code, lino, _end);





×







494













UNCOV

495



          if (record.suid != suid) {





×








UNCOV

496



            break;




×







497



          }










498













UNCOV

499



          px = taosArrayPush(pInfo->pUid, &record.uid);




×








UNCOV

500



          TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







501













UNCOV

502



          px = taosArrayPush(pInfo->pCount, &record.count);




×








UNCOV

503



          TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







504













UNCOV

505



          px = taosArrayPush(pInfo->pFirstTs, &record.firstKey.ts);




×








UNCOV

506



          TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







507













UNCOV

508



          px = taosArrayPush(pInfo->pLastTs, &record.lastKey.ts);




×








UNCOV

509



          TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







510













UNCOV

511



          if (record.firstKey.numOfPKs > 0) {





×







512



            SValue s = record.firstKey.pks[0];




×







513



            code = tValueDupPayload(&s);




×







514



            TSDB_CHECK_CODE(code, lino, _end);





×







515












516



            px = taosArrayPush(pInfo->pFirstKey, &s);




×







517



            TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







518












519



            s = record.lastKey.pks[0];




×







520



            code = tValueDupPayload(&s);




×







521



            TSDB_CHECK_CODE(code, lino, _end);





×







522












523



            px = taosArrayPush(pInfo->pLastKey, &s);




×







524



            TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







525



          } else {











UNCOV

526



            SValue v = {0};




×








UNCOV

527



            px = taosArrayPush(pInfo->pFirstKey, &v);




×








UNCOV

528



            TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







529













UNCOV

530



            px = taosArrayPush(pInfo->pLastKey, &v);




×








UNCOV

531



            TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







532



          }










533













UNCOV

534



          i += 1;




×







535



        }










536



      }










537



    }










538



  }










539













UNCOV

540



_end:




×








UNCOV

541



  tStatisBlockDestroy(&block);




×








UNCOV

542



  if (code != 0) {





×







543



    tsdbError("%s error happens at:%s line number: %d, code:%s", id, __func__, lino, tstrerror(code));





×







544



  } else {











UNCOV

545



    double el = (taosGetTimestampUs() - st) / 1000.0;




×








UNCOV

546



    pBlockLoadInfo->cost.statisElapsedTime += el;




×







547













UNCOV

548



    tsdbDebug("%s load %d statis blocks into buf, elapsed time:%.2fms", id, num, el);





×







549



  }











UNCOV

550



  return code;




×







551



}










552













UNCOV

553



static int32_t doLoadSttFilesBlk(SSttBlockLoadInfo *pBlockLoadInfo, SLDataIter *pIter, int64_t suid,




×







554



                                 _load_tomb_fn loadTombFn, void *pReader1, const char *idStr) {











UNCOV

555



  int64_t st = taosGetTimestampUs();




×







556













UNCOV

557



  const TSttBlkArray *pSttBlkArray = NULL;




×








UNCOV

558



  pBlockLoadInfo->sttBlockLoaded = true;




×







559












560



  // load the stt block info for each stt-block











UNCOV

561



  int32_t code = tsdbSttFileReadSttBlk(pIter->pReader, &pSttBlkArray);




×








UNCOV

562



  if (code != TSDB_CODE_SUCCESS) {





×







563



    tsdbError("load stt blk failed, code:%s, %s", tstrerror(code), idStr);





×







564



    return code;




×







565



  }










566












567



  // load the stt block info for each stt file block











UNCOV

568



  code = extractSttBlockInfo(pIter, pSttBlkArray, pBlockLoadInfo, suid);




×








UNCOV

569



  if (code != TSDB_CODE_SUCCESS) {





×







570



    tsdbError("load stt block info failed, code:%s, %s", tstrerror(code), idStr);





×







571



    return code;




×







572



  }










573












574



  // load stt statistics block for all stt-blocks, to decide if the data of queried table exists in current stt file











UNCOV

575



  TStatisBlkArray *pStatisBlkArray = NULL;




×








UNCOV

576



  code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray **)&pStatisBlkArray);




×








UNCOV

577



  if (code != TSDB_CODE_SUCCESS) {





×







578



    tsdbError("failed to load stt block statistics, code:%s, %s", tstrerror(code), idStr);





×







579



    return code;




×







580



  }










581












582



  // load statistics block for all tables in current stt file











UNCOV

583



  code = loadSttStatisticsBlockData(pIter->pReader, pIter->pBlockLoadInfo, pStatisBlkArray, suid, idStr);




×








UNCOV

584



  if (code != TSDB_CODE_SUCCESS) {





×







585



    tsdbError("failed to load stt statistics block data, code:%s, %s", tstrerror(code), idStr);





×







586



    return code;




×







587



  }










588













UNCOV

589



  code = loadTombFn(pReader1, pIter->pReader, pIter->pBlockLoadInfo);




×







590













UNCOV

591



  double el = (taosGetTimestampUs() - st) / 1000.0;




×








UNCOV

592



  tsdbDebug("load the stt file blk info completed, elapsed time:%.2fms, %s", el, idStr);





×








UNCOV

593



  return code;




×







594



}










595













UNCOV

596



static int32_t uidComparFn(const void *p1, const void *p2) {




×








UNCOV

597



  const uint64_t *pFirst = p1;




×








UNCOV

598



  const uint64_t *pVal = p2;




×







599













UNCOV

600



  if (*pFirst == *pVal) {





×








UNCOV

601



    return 0;




×







602



  } else {











UNCOV

603



    return *pFirst < *pVal ? -1 : 1;





×







604



  }










605



}










606













UNCOV

607



static void setSttInfoForCurrentTable(SSttBlockLoadInfo *pLoadInfo, uint64_t uid, SSttKeyRange *pRange,




×







608



                                      int64_t *numOfRows) {











UNCOV

609



  if (pRange == NULL || taosArrayGetSize(pLoadInfo->info.pUid) == 0) {





×








UNCOV

610



    return;




×







611



  }










612













UNCOV

613



  int32_t index = taosArraySearchIdx(pLoadInfo->info.pUid, &uid, uidComparFn, TD_EQ);




×








UNCOV

614



  if (index >= 0) {





×








UNCOV

615



    pRange->skey.ts = *(int64_t *)taosArrayGet(pLoadInfo->info.pFirstTs, index);




×








UNCOV

616



    pRange->ekey.ts = *(int64_t *)taosArrayGet(pLoadInfo->info.pLastTs, index);




×







617













UNCOV

618



    *numOfRows += *(int64_t *)taosArrayGet(pLoadInfo->info.pCount, index);




×







619













UNCOV

620



    if (pRange->skey.numOfPKs > 0) {





×








UNCOV

621



      memcpy(&pRange->skey.pks[0], taosArrayGet(pLoadInfo->info.pFirstKey, index), sizeof(SValue));




×








UNCOV

622



      memcpy(&pRange->ekey.pks[0], taosArrayGet(pLoadInfo->info.pLastKey, index), sizeof(SValue));




×







623



    }










624



  }










625



}










626













UNCOV

627



int32_t tLDataIterOpen2(SLDataIter *pIter, SSttFileReader *pSttFileReader, int32_t cid, int8_t backward,




×







628



                        SMergeTreeConf *pConf, SSttBlockLoadInfo *pBlockLoadInfo, SSttKeyRange *pKeyRange,










629



                        int64_t *numOfRows, const char *idStr) {











UNCOV

630



  int32_t code = TSDB_CODE_SUCCESS;




×







631













UNCOV

632



  pIter->uid = pConf->uid;




×








UNCOV

633



  pIter->cid = cid;




×








UNCOV

634



  pIter->backward = backward;




×








UNCOV

635



  pIter->verRange.minVer = pConf->verRange.minVer;




×








UNCOV

636



  pIter->verRange.maxVer = pConf->verRange.maxVer;




×








UNCOV

637



  pIter->timeWindow.skey = pConf->timewindow.skey;




×








UNCOV

638



  pIter->timeWindow.ekey = pConf->timewindow.ekey;




×







639













UNCOV

640



  pIter->pStartRowKey = pConf->pCurRowKey;




×








UNCOV

641



  pIter->pReader = pSttFileReader;




×








UNCOV

642



  pIter->pBlockLoadInfo = pBlockLoadInfo;




×







643












644



  // open stt file failed, ignore and continue











UNCOV

645



  if (pIter->pReader == NULL) {





×







646



    tsdbError("stt file reader is null, %s", idStr);





×







647



    pIter->pSttBlk = NULL;




×







648



    pIter->iSttBlk = -1;




×







649



    return TSDB_CODE_SUCCESS;




×







650



  }










651













UNCOV

652



  if (!pBlockLoadInfo->sttBlockLoaded) {





×








UNCOV

653



    code = doLoadSttFilesBlk(pBlockLoadInfo, pIter, pConf->suid, pConf->loadTombFn, pConf->pReader, idStr);




×








UNCOV

654



    if (code != TSDB_CODE_SUCCESS) {





×







655



      return code;




×







656



    }










657



  }










658













UNCOV

659



  setSttInfoForCurrentTable(pBlockLoadInfo, pConf->uid, pKeyRange, numOfRows);




×







660












661



  // find the start block, actually we could load the position to avoid repeatly searching for the start position when










662



  // the skey is updated.











UNCOV

663



  size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk);




×








UNCOV

664



  pIter->iSttBlk = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, pConf->uid, backward);




×








UNCOV

665



  if (pIter->iSttBlk != -1) {





×








UNCOV

666



    pIter->pSttBlk = taosArrayGet(pBlockLoadInfo->aSttBlk, pIter->iSttBlk);




×








UNCOV

667



    pIter->iRow = (pIter->backward) ? pIter->pSttBlk->nRow : -1;





×







668













UNCOV

669



    if ((!backward) && ((pConf->strictTimeRange && pIter->pSttBlk->minKey >= pIter->timeWindow.ekey) ||





×








UNCOV

670



                        (!pConf->strictTimeRange && pIter->pSttBlk->minKey > pIter->timeWindow.ekey))) {





×








UNCOV

671



      pIter->pSttBlk = NULL;




×







672



    }










673













UNCOV

674



    if (backward && ((pConf->strictTimeRange && pIter->pSttBlk->maxKey <= pIter->timeWindow.skey) ||





×








UNCOV

675



                     (!pConf->strictTimeRange && pIter->pSttBlk->maxKey < pIter->timeWindow.skey))) {





×








UNCOV

676



      pIter->pSttBlk = NULL;




×








UNCOV

677



      pIter->ignoreEarlierTs = true;




×







678



    }










679



  }










680













UNCOV

681



  return code;




×







682



}










683













UNCOV

684



void tLDataIterClose2(SLDataIter *pIter) {




×








UNCOV

685



  tsdbSttFileReaderClose(&pIter->pReader);




×








UNCOV

686



  pIter->pReader = NULL;




×








UNCOV

687



}




×







688













UNCOV

689



void tLDataIterNextBlock(SLDataIter *pIter, const char *idStr) {




×








UNCOV

690



  int32_t step = pIter->backward ? -1 : 1;





×








UNCOV

691



  int32_t oldIndex = pIter->iSttBlk;




×







692













UNCOV

693



  pIter->iSttBlk += step;




×







694













UNCOV

695



  int32_t index = -1;




×








UNCOV

696



  size_t  size = pIter->pBlockLoadInfo->aSttBlk->size;




×








UNCOV

697



  for (int32_t i = pIter->iSttBlk; i < size && i >= 0; i += step) {





×








UNCOV

698



    SSttBlk *p = taosArrayGet(pIter->pBlockLoadInfo->aSttBlk, i);




×








UNCOV

699



    if ((!pIter->backward) && p->minUid > pIter->uid) {





×








UNCOV

700



      break;




×







701



    }










702













UNCOV

703



    if (pIter->backward && p->maxUid < pIter->uid) {





×








UNCOV

704



      break;




×







705



    }










706












707



    // check uid firstly











UNCOV

708



    if (p->minUid <= pIter->uid && p->maxUid >= pIter->uid) {





×








UNCOV

709



      if ((!pIter->backward) && p->minKey > pIter->timeWindow.ekey) {





×








UNCOV

710



        break;




×







711



      }










712













UNCOV

713



      if (pIter->backward && p->maxKey < pIter->timeWindow.skey) {





×








UNCOV

714



        break;




×







715



      }










716












717



      // check time range secondly











UNCOV

718



      if (p->minKey <= pIter->timeWindow.ekey && p->maxKey >= pIter->timeWindow.skey) {





×








UNCOV

719



        if ((!pIter->backward) && p->minVer > pIter->verRange.maxVer) {





×







720



          break;




×







721



        }










722













UNCOV

723



        if (pIter->backward && p->maxVer < pIter->verRange.minVer) {





×







724



          break;




×







725



        }










726













UNCOV

727



        if (p->minVer <= pIter->verRange.maxVer && p->maxVer >= pIter->verRange.minVer) {





×








UNCOV

728



          index = i;




×








UNCOV

729



          break;




×







730



        }










731



      }










732



    }










733



  }










734













UNCOV

735



  pIter->pSttBlk = NULL;




×








UNCOV

736



  if (index != -1) {





×








UNCOV

737



    SSttBlk *p = taosArrayGet(pIter->pBlockLoadInfo->aSttBlk, index);




×







738













UNCOV

739



    pIter->iSttBlk = index;




×








UNCOV

740



    pIter->pSttBlk = (SSttBlk *)taosArrayGet(pIter->pBlockLoadInfo->aSttBlk, pIter->iSttBlk);




×








UNCOV

741



    tsdbDebug("try next stt-file block:%d from %d, trigger by uid:%" PRIu64 ", stt-fileVer:%" PRId64





×







742



              ", uidRange:%" PRId64 "-%" PRId64 " %s",










743



              pIter->iSttBlk, oldIndex, pIter->uid, pIter->cid, p->minUid, p->maxUid, idStr);










744



  } else {











UNCOV

745



    tsdbDebug("no more last block qualified, uid:%" PRIu64 ", stt-file block:%d, %s", pIter->uid, oldIndex, idStr);





×







746



  }











UNCOV

747



}




×







748













UNCOV

749



static int32_t findNextValidRow(SLDataIter *pIter, const char *idStr) {




×








UNCOV

750



  bool        hasVal = false;




×








UNCOV

751



  int32_t     step = pIter->backward ? -1 : 1;





×








UNCOV

752



  int32_t     i = pIter->iRow;




×








UNCOV

753



  SBlockData *pData = NULL;




×







754













UNCOV

755



  int32_t code = loadLastBlock(pIter, idStr, &pData);




×








UNCOV

756



  if (code) {





×







757



    tsdbError("failed to load stt block, code:%s, %s", tstrerror(code), idStr);





×







758



    return code;




×







759



  }










760












761



  // mostly we only need to find the start position for a given table











UNCOV

762



  if ((((i == 0) && (!pIter->backward)) || (i == pData->nRow - 1 && pIter->backward)) && pData->aUid != NULL) {





×








UNCOV

763



    i = binarySearchForStartRowIndex((uint64_t *)pData->aUid, pData->nRow, pIter->uid, pIter->backward);




×








UNCOV

764



    if (i == -1) {





×







765



      tsdbDebug("failed to find the data in pBlockData, uid:%" PRIu64 " , %s", pIter->uid, idStr);





×








UNCOV

766



      pIter->iRow = -1;




×








UNCOV

767



      return code;




×







768



    }










769



  }










770













UNCOV

771



  for (; i < pData->nRow && i >= 0; i += step) {





×








UNCOV

772



    if (pData->aUid != NULL) {





×








UNCOV

773



      if (!pIter->backward) {





×








UNCOV

774



        if (pData->aUid[i] > pIter->uid) {





×








UNCOV

775



          break;




×







776



        }










777



      } else {











UNCOV

778



        if (pData->aUid[i] < pIter->uid) {





×








UNCOV

779



          break;




×







780



        }










781



      }










782



    }










783













UNCOV

784



    int64_t ts = pData->aTSKEY[i];




×








UNCOV

785



    if (!pIter->backward) {               // asc





×








UNCOV

786



      if (ts > pIter->timeWindow.ekey) {  // no more data





×








UNCOV

787



        break;




×







788



      } else {











UNCOV

789



        if (ts < pIter->timeWindow.skey) {





×








UNCOV

790



          continue;




×







791



        }










792













UNCOV

793



        if (ts == pIter->timeWindow.skey && pIter->pStartRowKey->numOfPKs > 0) {





×







794



          SRowKey key;











UNCOV

795



          tColRowGetKey(pData, i, &key);





×








UNCOV

796



          int32_t ret = pkCompEx(&key, pIter->pStartRowKey);




×








UNCOV

797



          if (ret < 0) {





×







798



            continue;




×







799



          }










800



        }










801



      }










802



    } else {











UNCOV

803



      if (ts < pIter->timeWindow.skey) {





×








UNCOV

804



        break;




×







805



      } else {











UNCOV

806



        if (ts > pIter->timeWindow.ekey) {





×








UNCOV

807



          continue;




×







808



        }










809













UNCOV

810



        if (ts == pIter->timeWindow.ekey && pIter->pStartRowKey->numOfPKs > 0) {





×







811



          SRowKey key;











UNCOV

812



          tColRowGetKey(pData, i, &key);





×








UNCOV

813



          int32_t ret = pkCompEx(&key, pIter->pStartRowKey);




×








UNCOV

814



          if (ret > 0) {





×







815



            continue;




×







816



          }










817



        }










818



      }










819



    }










820













UNCOV

821



    int64_t ver = pData->aVersion[i];




×








UNCOV

822



    if (ver < pIter->verRange.minVer) {





×







823



      continue;




×







824



    }










825












826



    // todo opt handle desc case











UNCOV

827



    if (ver > pIter->verRange.maxVer) {





×








UNCOV

828



      continue;




×







829



    }










830













UNCOV

831



    hasVal = true;




×








UNCOV

832



    break;




×







833



  }










834













UNCOV

835



  pIter->iRow = (hasVal) ? i : -1;





×








UNCOV

836



  return code;




×







837



}










838













UNCOV

839



int32_t tLDataIterNextRow(SLDataIter *pIter, const char *idStr, bool *hasNext) {




×








UNCOV

840



  int32_t     step = pIter->backward ? -1 : 1;





×








UNCOV

841



  int32_t     code = 0;




×








UNCOV

842



  int32_t     iBlockL = pIter->iSttBlk;




×








UNCOV

843



  SBlockData *pBlockData = NULL;




×








UNCOV

844



  int32_t     lino = 0;




×







845













UNCOV

846



  *hasNext = false;




×







847












848



  // no qualified last file block in current file, no need to fetch row











UNCOV

849



  if (pIter->pSttBlk == NULL) {





×








UNCOV

850



    return code;




×







851



  }










852













UNCOV

853



  code = loadLastBlock(pIter, idStr, &pBlockData);




×








UNCOV

854



  if (pBlockData == NULL || code != TSDB_CODE_SUCCESS) {





×







855



    lino = __LINE__;




×







856



    goto _exit;




×







857



  }










858













UNCOV

859



  pIter->iRow += step;




×







860













UNCOV

861



  while (1) {




×








UNCOV

862



    bool skipBlock = false;




×








UNCOV

863



    code = findNextValidRow(pIter, idStr);




×








UNCOV

864



    TSDB_CHECK_CODE(code, lino, _exit);





×







865













UNCOV

866



    if (pIter->pBlockLoadInfo->checkRemainingRow) {





×







867



      skipBlock = true;




×







868



      int16_t *aCols = pIter->pBlockLoadInfo->colIds;




×







869



      int      nCols = pIter->pBlockLoadInfo->numOfCols;




×







870



      bool     isLast = pIter->pBlockLoadInfo->isLast;




×







871



      for (int inputColIndex = 0; inputColIndex < nCols; ++inputColIndex) {





×







872



        for (int colIndex = 0; colIndex < pBlockData->nColData; ++colIndex) {





×







873



          SColData *pColData = &pBlockData->aColData[colIndex];




×







874



          int16_t   cid = pColData->cid;




×







875












876



          if (cid == aCols[inputColIndex]) {





×







877



            if (isLast && (pColData->flag & HAS_VALUE)) {





×







878



              skipBlock = false;




×







879



              break;




×







880



            } else if (pColData->flag & (HAS_VALUE | HAS_NULL)) {





×







881



              skipBlock = false;




×







882



              break;




×







883



            }










884



          }










885



        }










886



      }










887



    }










888













UNCOV

889



    if (skipBlock || pIter->iRow >= pBlockData->nRow || pIter->iRow < 0) {





×








UNCOV

890



      tLDataIterNextBlock(pIter, idStr);




×








UNCOV

891



      if (pIter->pSttBlk == NULL) {  // no more data





×








UNCOV

892



        goto _exit;




×







893



      }










894



    } else {










895



      break;










896



    }










897













UNCOV

898



    if (iBlockL != pIter->iSttBlk) {





×








UNCOV

899



      code = loadLastBlock(pIter, idStr, &pBlockData);




×








UNCOV

900



      if ((pBlockData == NULL) || (code != 0)) {





×







901



        lino = __LINE__;




×







902



        goto _exit;




×







903



      }










904












905



      // set start row index











UNCOV

906



      pIter->iRow = pIter->backward ? pBlockData->nRow - 1 : 0;





×







907



    }










908



  }










909













UNCOV

910



  pIter->rInfo.suid = pBlockData->suid;




×








UNCOV

911



  pIter->rInfo.uid = pBlockData->uid;




×








UNCOV

912



  pIter->rInfo.row = tsdbRowFromBlockData(pBlockData, pIter->iRow);




×







913













UNCOV

914



_exit:




×








UNCOV

915



  if (code) {





×







916



    tsdbError("failed to exec stt-file nextIter, lino:%d, code:%s, %s", lino, tstrerror(code), idStr);





×







917



  }










918













UNCOV

919



  *hasNext = (code == TSDB_CODE_SUCCESS) && (pIter->pSttBlk != NULL) && (pBlockData != NULL);





×








UNCOV

920



  return code;




×







921



}










922












923



// SMergeTree =================================================











UNCOV

924



static FORCE_INLINE int32_t tLDataIterCmprFn(const SRBTreeNode *p1, const SRBTreeNode *p2) {




×








UNCOV

925



  SLDataIter *pIter1 = (SLDataIter *)(((uint8_t *)p1) - offsetof(SLDataIter, node));




×








UNCOV

926



  SLDataIter *pIter2 = (SLDataIter *)(((uint8_t *)p2) - offsetof(SLDataIter, node));




×







927













UNCOV

928



  SRowKey rkey1 = {0}, rkey2 = {0};




×








UNCOV

929



  tRowGetKeyEx(&pIter1->rInfo.row, &rkey1);





×








UNCOV

930



  tRowGetKeyEx(&pIter2->rInfo.row, &rkey2);





×







931













UNCOV

932



  int32_t ret = tRowKeyCompare(&rkey1, &rkey2);




×








UNCOV

933



  if (ret < 0) {





×








UNCOV

934



    return -1;




×








UNCOV

935



  } else if (ret > 0) {





×








UNCOV

936



    return 1;




×







937



  } else {











UNCOV

938



    int64_t ver1 = TSDBROW_VERSION(&pIter1->rInfo.row);





×








UNCOV

939



    int64_t ver2 = TSDBROW_VERSION(&pIter2->rInfo.row);





×







940













UNCOV

941



    if (ver1 < ver2) {





×








UNCOV

942



      return -1;




×








UNCOV

943



    } else if (ver1 > ver2) {





×








UNCOV

944



      return 1;




×







945



    } else {










946



      return 0;




×







947



    }










948



  }










949



}










950













UNCOV

951



static FORCE_INLINE int32_t tLDataIterDescCmprFn(const SRBTreeNode *p1, const SRBTreeNode *p2) {





×








UNCOV

952



  return -1 * tLDataIterCmprFn(p1, p2);




×







953



}










954













UNCOV

955



int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoForTable *pSttDataInfo) {




×








UNCOV

956



  int32_t code = TSDB_CODE_SUCCESS;




×







957













UNCOV

958



  pMTree->pIter = NULL;




×








UNCOV

959



  pMTree->backward = pConf->backward;




×








UNCOV

960



  pMTree->idStr = pConf->idstr;




×








UNCOV

961



  int32_t lino = 0;




×







962













UNCOV

963



  if (!pMTree->backward) {  // asc





×








UNCOV

964



    tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn);




×







965



  } else {  // desc











UNCOV

966



    tRBTreeCreate(&pMTree->rbt, tLDataIterDescCmprFn);




×







967



  }










968













UNCOV

969



  pMTree->ignoreEarlierTs = false;




×







970












971



  // no data exists, go to end











UNCOV

972



  int32_t numOfLevels = ((STFileSet *)pConf->pCurrentFileset)->lvlArr->size;




×








UNCOV

973



  if (numOfLevels == 0) {





×








UNCOV

974



    goto _end;




×







975



  }










976













UNCOV

977



  code = adjustSttDataIters(pConf->pSttFileBlockIterArray, pConf->pCurrentFileset);




×








UNCOV

978



  if (code) {





×







979



    goto _end;




×







980



  }










981













UNCOV

982



  for (int32_t j = 0; j < numOfLevels; ++j) {





×








UNCOV

983



    SSttLvl *pSttLevel = ((STFileSet *)pConf->pCurrentFileset)->lvlArr->data[j];




×








UNCOV

984



    SArray  *pList = taosArrayGetP(pConf->pSttFileBlockIterArray, j);




×







985













UNCOV

986



    for (int32_t i = 0; i < TARRAY2_SIZE(pSttLevel->fobjArr); ++i) {  // open all last file





×








UNCOV

987



      SLDataIter *pIter = taosArrayGetP(pList, i);




×







988













UNCOV

989



      SSttFileReader    *pSttFileReader = pIter->pReader;




×








UNCOV

990



      SSttBlockLoadInfo *pLoadInfo = pIter->pBlockLoadInfo;




×







991












992



      // open stt file reader if not opened yet










993



      // if failed to open this stt file, ignore the error and try next one











UNCOV

994



      if (pSttFileReader == NULL) {





×








UNCOV

995



        SSttFileReaderConfig conf = {.tsdb = pConf->pTsdb, .szPage = pConf->pTsdb->pVnode->config.tsdbPageSize};




×








UNCOV

996



        conf.file[0] = *pSttLevel->fobjArr->data[i]->f;




×







997













UNCOV

998



        code = tsdbSttFileReaderOpen(pSttLevel->fobjArr->data[i]->fname, &conf, &pSttFileReader);




×








UNCOV

999



        if (code != TSDB_CODE_SUCCESS) {





×







1000



          tsdbError("open stt file reader error. file name %s, code %s, %s", pSttLevel->fobjArr->data[i]->fname,





×







1001



                    tstrerror(code), pMTree->idStr);










1002



        }










1003



      }










1004













UNCOV

1005



      if (pLoadInfo == NULL) {





×








UNCOV

1006



        code = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols, &pLoadInfo);




×








UNCOV

1007



        if (code != TSDB_CODE_SUCCESS) {





×







1008



          goto _end;




×







1009



        }










1010



      }










1011













UNCOV

1012



      memset(pIter, 0, sizeof(SLDataIter));




×







1013













UNCOV

1014



      SSttKeyRange range = {.skey.numOfPKs = pConf->pCurRowKey->numOfPKs, .ekey.numOfPKs = pConf->pCurRowKey->numOfPKs};




×








UNCOV

1015



      int64_t      numOfRows = 0;




×








UNCOV

1016



      int64_t      cid = pSttLevel->fobjArr->data[i]->f->cid;




×







1017













UNCOV

1018



      code = tLDataIterOpen2(pIter, pSttFileReader, cid, pMTree->backward, pConf, pLoadInfo, &range, &numOfRows,




×







1019



                             pMTree->idStr);











UNCOV

1020



      if (code != TSDB_CODE_SUCCESS) {





×







1021



        goto _end;




×







1022



      }










1023













UNCOV

1024



      bool hasVal = NULL;




×








UNCOV

1025



      code = tLDataIterNextRow(pIter, pMTree->idStr, &hasVal);




×








UNCOV

1026



      if (code) {





×







1027



        goto _end;




×







1028



      }










1029













UNCOV

1030



      if (hasVal) {





×








UNCOV

1031



        tMergeTreeAddIter(pMTree, pIter);




×







1032












1033



        // let's record the time window for current table of uid in the stt files











UNCOV

1034



        if (pSttDataInfo != NULL && numOfRows > 0) {





×








UNCOV

1035



          void *px = taosArrayPush(pSttDataInfo->pKeyRangeList, &range);




×








UNCOV

1036



          QUERY_CHECK_NULL(px, code, lino, _end, terrno);





×







1037













UNCOV

1038



          pSttDataInfo->numOfRows += numOfRows;




×







1039



        }










1040



      } else {











UNCOV

1041



        if (!pMTree->ignoreEarlierTs) {





×








UNCOV

1042



          pMTree->ignoreEarlierTs = pIter->ignoreEarlierTs;




×







1043



        }










1044



      }










1045



    }










1046



  }










1047













UNCOV

1048



  return code;




×







1049













UNCOV

1050



_end:




×








UNCOV

1051



  tMergeTreeClose(pMTree);




×








UNCOV

1052



  return code;




×







1053



}










1054













UNCOV

1055



void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter) {




×








UNCOV

1056



  SRBTreeNode *node = tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pIter);




×








UNCOV

1057



}




×







1058












1059



bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree) { return pMTree->ignoreEarlierTs; }




×







1060













UNCOV

1061



static void tLDataIterPinSttBlock(SLDataIter *pIter, const char *id) {




×








UNCOV

1062



  SSttBlockLoadInfo *pInfo = pIter->pBlockLoadInfo;




×







1063













UNCOV

1064



  if (pInfo->blockData[0].sttBlockIndex == pIter->iSttBlk) {





×








UNCOV

1065



    pInfo->blockData[0].pin = true;




×








UNCOV

1066



    tsdbTrace("pin stt-block, blockIndex:%d, stt-fileVer:%" PRId64 " %s", pIter->iSttBlk, pIter->cid, id);





×








UNCOV

1067



    return;




×







1068



  }










1069













UNCOV

1070



  if (pInfo->blockData[1].sttBlockIndex == pIter->iSttBlk) {





×








UNCOV

1071



    pInfo->blockData[1].pin = true;




×








UNCOV

1072



    tsdbTrace("pin stt-block, blockIndex:%d, stt-fileVer:%" PRId64 " %s", pIter->iSttBlk, pIter->cid, id);





×








UNCOV

1073



    return;




×







1074



  }










1075












1076



  tsdbError("failed to pin any stt block, sttBlock:%d stt-fileVer:%" PRId64 " %s", pIter->iSttBlk, pIter->cid, id);





×







1077



}










1078













UNCOV

1079



static void tLDataIterUnpinSttBlock(SLDataIter *pIter, const char *id) {




×








UNCOV

1080



  SSttBlockLoadInfo *pInfo = pIter->pBlockLoadInfo;




×








UNCOV

1081



  if (pInfo->blockData[0].pin) {





×








UNCOV

1082



    pInfo->blockData[0].pin = false;




×








UNCOV

1083



    tsdbTrace("unpin stt-block:%d, stt-fileVer:%" PRId64 " %s", pInfo->blockData[0].sttBlockIndex, pIter->cid, id);





×








UNCOV

1084



    return;




×







1085



  }










1086













UNCOV

1087



  if (pInfo->blockData[1].pin) {





×








UNCOV

1088



    pInfo->blockData[1].pin = false;




×








UNCOV

1089



    tsdbTrace("unpin stt-block:%d, stt-fileVer:%" PRId64 " %s", pInfo->blockData[1].sttBlockIndex, pIter->cid, id);





×








UNCOV

1090



    return;




×







1091



  }










1092












1093



  tsdbError("failed to unpin any stt block, sttBlock:%d stt-fileVer:%" PRId64 " %s", pIter->iSttBlk, pIter->cid, id);





×







1094



}










1095













UNCOV

1096



void tMergeTreePinSttBlock(SMergeTree *pMTree) {




×








UNCOV

1097



  if (pMTree->pIter == NULL) {





×







1098



    return;




×







1099



  }










1100













UNCOV

1101



  SLDataIter *pIter = pMTree->pIter;




×








UNCOV

1102



  pMTree->pPinnedBlockIter = pIter;




×








UNCOV

1103



  tLDataIterPinSttBlock(pIter, pMTree->idStr);




×







1104



}










1105













UNCOV

1106



void tMergeTreeUnpinSttBlock(SMergeTree *pMTree) {




×








UNCOV

1107



  if (pMTree->pPinnedBlockIter == NULL) {





×







1108



    return;




×







1109



  }










1110













UNCOV

1111



  SLDataIter *pIter = pMTree->pPinnedBlockIter;




×








UNCOV

1112



  pMTree->pPinnedBlockIter = NULL;




×








UNCOV

1113



  tLDataIterUnpinSttBlock(pIter, pMTree->idStr);




×







1114



}










1115













UNCOV

1116



int32_t tMergeTreeNext(SMergeTree *pMTree, bool *pHasNext) {




×








UNCOV

1117



  int32_t code = 0;




×








UNCOV

1118



  if (pHasNext == NULL) {





×







1119



    return TSDB_CODE_INVALID_PARA;




×







1120



  }










1121













UNCOV

1122



  *pHasNext = false;




×








UNCOV

1123



  if (pMTree->pIter) {





×








UNCOV

1124



    SLDataIter *pIter = pMTree->pIter;




×








UNCOV

1125



    bool        hasVal = false;




×








UNCOV

1126



    code = tLDataIterNextRow(pIter, pMTree->idStr, &hasVal);




×








UNCOV

1127



    if (!hasVal || (code != 0)) {





×








UNCOV

1128



      if (code == TSDB_CODE_FILE_CORRUPTED) {





×







1129



        code = 0;  // suppress the file corrupt error to enable all queries within this cluster can run without failed.




×







1130



      }










1131













UNCOV

1132



      pMTree->pIter = NULL;




×







1133



    }










1134












1135



    // compare with min in RB Tree











UNCOV

1136



    pIter = (SLDataIter *)tRBTreeMin(&pMTree->rbt);





×








UNCOV

1137



    if (pMTree->pIter && pIter) {





×








UNCOV

1138



      int32_t c = pMTree->rbt.cmprFn(&pMTree->pIter->node, &pIter->node);




×








UNCOV

1139



      if (c > 0) {





×








UNCOV

1140



        (void)tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pMTree->pIter);




×








UNCOV

1141



        pMTree->pIter = NULL;




×








UNCOV

1142



      } else if (!c) {





×







1143



        return TSDB_CODE_INTERNAL_ERROR;




×







1144



      }










1145



    }










1146



  }










1147













UNCOV

1148



  if (pMTree->pIter == NULL) {





×








UNCOV

1149



    pMTree->pIter = (SLDataIter *)tRBTreeMin(&pMTree->rbt);





×








UNCOV

1150



    if (pMTree->pIter) {





×








UNCOV

1151



      tRBTreeDrop(&pMTree->rbt, (SRBTreeNode *)pMTree->pIter);




×







1152



    }










1153



  }










1154













UNCOV

1155



  *pHasNext = (pMTree->pIter != NULL);




×








UNCOV

1156



  return code;




×







1157



}










1158













UNCOV

1159



void tMergeTreeClose(SMergeTree *pMTree) {




×








UNCOV

1160



  pMTree->pIter = NULL;




×








UNCOV

1161



  pMTree->pPinnedBlockIter = NULL;




×








UNCOV

1162



}




×





















    

  • 

    •