taosdata / TDengine / #3565

coverage: 51.098% (-11.1%) from 62.21%
#3565

push

travis-ci

web-flow 
Merge pull request #29316 from taosdata/enh/3.0/TD-33266

enh(ut):Add wal & config UT.

111558 of 284773 branches covered (39.17%)

Branch coverage included in aggregate %.

1 of 2 new or added lines in 2 files covered. (50.0%)

39015 existing lines in 102 files now uncovered.

177882 of 281666 relevant lines covered (63.15%)

15090998.35 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/source/dnode/vnode/src/tsdb/tsdbReadUtil.c
1 
/*
2 
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3 
 *
4 
 * This program is free software: you can use, redistribute, and/or modify
5 
 * it under the terms of the GNU Affero General Public License, version 3
6 
 * or later ("AGPL"), as published by the Free Software Foundation.
7 
 *
8 
 * This program is distributed in the hope that it will be useful, but WITHOUT
9 
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 
 * FITNESS FOR A PARTICULAR PURPOSE.
11 
 *
12 
 * You should have received a copy of the GNU Affero General Public License
13 
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
14 
 */
15 









16



#include "tsdbReadUtil.h"










17



#include "tsdb.h"










18



#include "tsdbDataFileRW.h"










19



#include "tsdbFS2.h"










20



#include "tsdbMerge.h"










21



#include "tsdbUtil2.h"










22



#include "tsimplehash.h"










23












24



static bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord,










25



                                            int32_t order);










26













UNCOV

27



static int32_t initBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) {




×








UNCOV

28



  int32_t              code = TSDB_CODE_SUCCESS;




×








UNCOV

29



  int32_t              lino = 0;




×








UNCOV

30



  int32_t              num = 0;




×








UNCOV

31



  int32_t              remainder = 0;




×








UNCOV

32



  STableBlockScanInfo* p = NULL;




×








UNCOV

33



  const void*          px = NULL;




×







34













UNCOV

35



  TSDB_CHECK_CONDITION(pBuf && pBuf->numPerBucket > 0, code, lino, _end, TSDB_CODE_INVALID_PARA);





×








UNCOV

36



  TSDB_CHECK_CONDITION(numOfTables >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA);





×







37













UNCOV

38



  num = numOfTables / pBuf->numPerBucket;




×








UNCOV

39



  remainder = numOfTables % pBuf->numPerBucket;




×







40













UNCOV

41



  if (pBuf->pData == NULL) {





×








UNCOV

42



    pBuf->pData = taosArrayInit(num + 1, POINTER_BYTES);




×








UNCOV

43



    TSDB_CHECK_NULL(pBuf->pData, code, lino, _end, terrno);





×







44



  }










45













UNCOV

46



  for (int32_t i = 0; i < num; ++i) {





×








UNCOV

47



    p = taosMemoryCalloc(pBuf->numPerBucket, sizeof(STableBlockScanInfo));





×








UNCOV

48



    TSDB_CHECK_NULL(p, code, lino, _end, terrno);





×







49













UNCOV

50



    px = taosArrayPush(pBuf->pData, &p);




×








UNCOV

51



    TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×








UNCOV

52



    p = NULL;




×







53



  }










54













UNCOV

55



  if (remainder > 0) {





×








UNCOV

56



    p = taosMemoryCalloc(remainder, sizeof(STableBlockScanInfo));





×








UNCOV

57



    TSDB_CHECK_NULL(p, code, lino, _end, terrno);





×







58













UNCOV

59



    px = taosArrayPush(pBuf->pData, &p);




×








UNCOV

60



    TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×








UNCOV

61



    p = NULL;




×







62



  }










63













UNCOV

64



  pBuf->numOfTables = numOfTables;




×







65













UNCOV

66



_end:




×








UNCOV

67



  if (code != TSDB_CODE_SUCCESS) {





×







68



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));





×







69



  }











UNCOV

70



  if (p) {





×







71



    taosMemoryFreeClear(p);





×







72



  }











UNCOV

73



  return code;




×







74



}










75













UNCOV

76



int32_t uidComparFunc(const void* p1, const void* p2) {




×








UNCOV

77



  uint64_t pu1 = *(const uint64_t*)p1;




×








UNCOV

78



  uint64_t pu2 = *(const uint64_t*)p2;




×








UNCOV

79



  if (pu1 == pu2) {





×







80



    return 0;




×







81



  } else {











UNCOV

82



    return (pu1 < pu2) ? -1 : 1;





×







83



  }










84



}










85













UNCOV

86



int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) {




×








UNCOV

87



  int32_t              code = TSDB_CODE_SUCCESS;




×








UNCOV

88



  int32_t              lino = 0;




×








UNCOV

89



  int32_t              num = 0;




×








UNCOV

90



  int32_t              remainder = 0;




×








UNCOV

91



  STableBlockScanInfo* p = NULL;




×








UNCOV

92



  const void*          px = NULL;




×







93













UNCOV

94



  TSDB_CHECK_CONDITION(pBuf && pBuf->numPerBucket > 0 && pBuf->numOfTables >= 0, code, lino, _end,





×







95



                       TSDB_CODE_INVALID_PARA);











UNCOV

96



  TSDB_CHECK_CONDITION(numOfTables >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA);





×







97













UNCOV

98



  if (numOfTables <= pBuf->numOfTables) {





×







99



    goto _end;




×







100



  }










101













UNCOV

102



  remainder = pBuf->numOfTables % pBuf->numPerBucket;




×








UNCOV

103



  if (remainder > 0) {





×








UNCOV

104



    TSDB_CHECK_CONDITION(taosArrayGetSize(pBuf->pData) > 0, code, lino, _end, TSDB_CODE_INVALID_PARA);





×








UNCOV

105



    px = taosArrayPop(pBuf->pData);




×








UNCOV

106



    TSDB_CHECK_NULL(px, code, lino, _end, TSDB_CODE_INVALID_PARA);





×








UNCOV

107



    p = *(STableBlockScanInfo**)px;




×








UNCOV

108



    taosMemoryFreeClear(p);





×








UNCOV

109



    pBuf->numOfTables -= remainder;




×







110



  }










111













UNCOV

112



  num = (numOfTables - pBuf->numOfTables) / pBuf->numPerBucket;




×








UNCOV

113



  remainder = (numOfTables - pBuf->numOfTables) % pBuf->numPerBucket;




×







114













UNCOV

115



  if (pBuf->pData == NULL) {





×







116



    pBuf->pData = taosArrayInit(num + 1, POINTER_BYTES);




×







117



    TSDB_CHECK_NULL(pBuf->pData, code, lino, _end, terrno);





×







118



  }










119













UNCOV

120



  for (int32_t i = 0; i < num; ++i) {





×







121



    p = taosMemoryCalloc(pBuf->numPerBucket, sizeof(STableBlockScanInfo));





×







122



    TSDB_CHECK_NULL(p, code, lino, _end, terrno);





×







123












124



    px = taosArrayPush(pBuf->pData, &p);




×







125



    TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







126



    p = NULL;




×







127



  }










128













UNCOV

129



  if (remainder > 0) {





×








UNCOV

130



    p = taosMemoryCalloc(remainder, sizeof(STableBlockScanInfo));





×








UNCOV

131



    TSDB_CHECK_NULL(p, code, lino, _end, terrno);





×







132













UNCOV

133



    px = taosArrayPush(pBuf->pData, &p);




×








UNCOV

134



    TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×








UNCOV

135



    p = NULL;




×







136



  }










137













UNCOV

138



  pBuf->numOfTables = numOfTables;




×







139













UNCOV

140



_end:




×








UNCOV

141



  if (code != TSDB_CODE_SUCCESS) {





×







142



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));





×







143



  }











UNCOV

144



  if (p) {





×







145



    taosMemoryFreeClear(p);





×







146



  }











UNCOV

147



  return code;




×







148



}










149













UNCOV

150



void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf) {




×








UNCOV

151



  if (pBuf == NULL) return;





×








UNCOV

152



  if (pBuf->pData != NULL) {





×








UNCOV

153



    taosArrayDestroyP(pBuf->pData, NULL);




×








UNCOV

154



    pBuf->pData = NULL;




×







155



  }










156



}










157













UNCOV

158



int32_t getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index, STableBlockScanInfo** pInfo) {




×








UNCOV

159



  int32_t               code = TSDB_CODE_SUCCESS;




×








UNCOV

160



  int32_t               lino = 0;




×








UNCOV

161



  int32_t               bucketIndex = 0;




×








UNCOV

162



  STableBlockScanInfo** pBucket = NULL;




×







163













UNCOV

164



  TSDB_CHECK_CONDITION(pBuf && pBuf->numPerBucket > 0, code, lino, _end, TSDB_CODE_INVALID_PARA);





×








UNCOV

165



  TSDB_CHECK_CONDITION(index >= 0 && index < pBuf->numOfTables, code, lino, _end, TSDB_CODE_INVALID_PARA);





×








UNCOV

166



  TSDB_CHECK_NULL(pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);





×







167













UNCOV

168



  *pInfo = NULL;




×







169













UNCOV

170



  bucketIndex = index / pBuf->numPerBucket;




×








UNCOV

171



  pBucket = taosArrayGet(pBuf->pData, bucketIndex);




×








UNCOV

172



  TSDB_CHECK_NULL(pBucket, code, lino, _end, terrno);





×







173













UNCOV

174



  *pInfo = (*pBucket) + (index % pBuf->numPerBucket);




×







175













UNCOV

176



_end:




×








UNCOV

177



  if (code != TSDB_CODE_SUCCESS) {





×







178



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));





×







179



  }











UNCOV

180



  return code;




×







181



}










182













UNCOV

183



int32_t getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, STableBlockScanInfo** pInfo, const char* id) {




×








UNCOV

184



  int32_t               code = TSDB_CODE_SUCCESS;




×








UNCOV

185



  int32_t               lino = 0;




×








UNCOV

186



  STableBlockScanInfo** pVal = NULL;




×







187













UNCOV

188



  TSDB_CHECK_NULL(pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);





×








UNCOV

189



  TSDB_CHECK_NULL(id, code, lino, _end, TSDB_CODE_INVALID_PARA);





×







190













UNCOV

191



  pVal = (STableBlockScanInfo**)tSimpleHashGet(pTableMap, &uid, sizeof(uid));




×








UNCOV

192



  if (pVal == NULL) {





×







193



    int32_t size = tSimpleHashGetSize(pTableMap);




×







194



    tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id);





×







195



    code = TSDB_CODE_INVALID_PARA;




×







196



    TSDB_CHECK_CODE(code, lino, _end);





×







197



  }











UNCOV

198



  *pInfo = *pVal;




×








UNCOV

199



  TSDB_CHECK_NULL(*pInfo, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);





×







200













UNCOV

201



_end:




×








UNCOV

202



  if (code != TSDB_CODE_SUCCESS) {





×







203



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));





×







204



  }











UNCOV

205



  return code;




×







206



}










207













UNCOV

208



int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc) {




×








UNCOV

209



  int32_t code = TSDB_CODE_SUCCESS;




×








UNCOV

210



  int32_t lino = 0;




×







211













UNCOV

212



  TSDB_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA);





×







213













UNCOV

214



  pKey->numOfPKs = numOfPks;




×








UNCOV

215



  pKey->ts = ts;




×







216













UNCOV

217



  if (numOfPks > 0) {





×








UNCOV

218



    pKey->pks[0].type = type;




×







219













UNCOV

220



    if (IS_NUMERIC_TYPE(type)) {





×








UNCOV

221



      if (asc) {





×








UNCOV

222



        switch (type) {





×








UNCOV

223



          case TSDB_DATA_TYPE_BIGINT: {




×








UNCOV

224



            pKey->pks[0].val = INT64_MIN;




×








UNCOV

225



            break;




×







226



          }











UNCOV

227



          case TSDB_DATA_TYPE_INT: {




×








UNCOV

228



            int32_t min = INT32_MIN;




×








UNCOV

229



            (void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);




×








UNCOV

230



            break;




×







231



          }










232



          case TSDB_DATA_TYPE_SMALLINT: {




×







233



            int16_t min = INT16_MIN;




×







234



            (void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);




×







235



            break;




×







236



          }










237



          case TSDB_DATA_TYPE_TINYINT: {




×







238



            int8_t min = INT8_MIN;




×







239



            (void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);




×







240



            break;




×







241



          }










242



          case TSDB_DATA_TYPE_UTINYINT:




×







243



          case TSDB_DATA_TYPE_USMALLINT:










244



          case TSDB_DATA_TYPE_UINT:










245



          case TSDB_DATA_TYPE_UBIGINT: {










246



            pKey->pks[0].val = 0;




×







247



            break;




×







248



          }










249



          default:




×







250



            code = TSDB_CODE_INVALID_PARA;




×







251



            TSDB_CHECK_CODE(code, lino, _end);





×







252



        }










253



      } else {











UNCOV

254



        switch (type) {





×








UNCOV

255



          case TSDB_DATA_TYPE_BIGINT:




×








UNCOV

256



            pKey->pks[0].val = INT64_MAX;




×








UNCOV

257



            break;




×








UNCOV

258



          case TSDB_DATA_TYPE_INT:




×








UNCOV

259



            pKey->pks[0].val = INT32_MAX;




×








UNCOV

260



            break;




×







261



          case TSDB_DATA_TYPE_SMALLINT:




×







262



            pKey->pks[0].val = INT16_MAX;




×







263



            break;




×







264



          case TSDB_DATA_TYPE_TINYINT:




×







265



            pKey->pks[0].val = INT8_MAX;




×







266



            break;




×







267



          case TSDB_DATA_TYPE_UBIGINT:




×







268



            pKey->pks[0].val = UINT64_MAX;




×







269



            break;




×







270



          case TSDB_DATA_TYPE_UINT:




×







271



            pKey->pks[0].val = UINT32_MAX;




×







272



            break;




×







273



          case TSDB_DATA_TYPE_USMALLINT:




×







274



            pKey->pks[0].val = UINT16_MAX;




×







275



            break;




×







276



          case TSDB_DATA_TYPE_UTINYINT:




×







277



            pKey->pks[0].val = UINT8_MAX;




×







278



            break;




×







279



          default:




×







280



            code = TSDB_CODE_INVALID_PARA;




×







281



            TSDB_CHECK_CODE(code, lino, _end);





×







282



        }










283



      }










284



    } else {











UNCOV

285



      pKey->pks[0].nData = 0;




×








UNCOV

286



      pKey->pks[0].pData = taosMemoryCalloc(1, len);





×








UNCOV

287



      TSDB_CHECK_NULL(pKey->pks[0].pData, code, lino, _end, terrno);





×







288













UNCOV

289



      if (!asc) {





×








UNCOV

290



        pKey->numOfPKs = 2;




×







291



      }










292



    }










293



  }










294













UNCOV

295



_end:




×








UNCOV

296



  if (code != TSDB_CODE_SUCCESS) {





×







297



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));





×







298



  }











UNCOV

299



  return code;




×







300



}










301













UNCOV

302



void clearRowKey(SRowKey* pKey) {




×








UNCOV

303



  if (pKey == NULL || pKey->numOfPKs == 0 || (!IS_VAR_DATA_TYPE(pKey->pks[0].type))) {





×








UNCOV

304



    return;




×







305



  }










306



  taosMemoryFreeClear(pKey->pks[0].pData);





×







307



}










308













UNCOV

309



static int32_t initLastProcKey(STableBlockScanInfo* pScanInfo, const STsdbReader* pReader) {




×








UNCOV

310



  int32_t  code = TSDB_CODE_SUCCESS;




×








UNCOV

311



  int32_t  lino = 0;




×








UNCOV

312



  int32_t  numOfPks = 0;




×








UNCOV

313



  int32_t  type = 0;




×








UNCOV

314



  int32_t  bytes = 0;




×








UNCOV

315



  bool     asc = false;




×








UNCOV

316



  SRowKey* pRowKey = NULL;




×







317













UNCOV

318



  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);





×








UNCOV

319



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);





×







320













UNCOV

321



  numOfPks = pReader->suppInfo.numOfPks;




×








UNCOV

322



  asc = ASCENDING_TRAVERSE(pReader->info.order);




×








UNCOV

323



  type = pReader->suppInfo.pk.type;




×








UNCOV

324



  bytes = pReader->suppInfo.pk.bytes;




×







325













UNCOV

326



  pRowKey = &pScanInfo->lastProcKey;




×








UNCOV

327



  if (asc) {





×








UNCOV

328



    int64_t skey = pReader->info.window.skey;




×








UNCOV

329



    int64_t ts = (skey > INT64_MIN) ? (skey - 1) : skey;





×







330













UNCOV

331



    code = initRowKey(pRowKey, ts, numOfPks, type, bytes, asc);




×








UNCOV

332



    TSDB_CHECK_CODE(code, lino, _end);





×







333













UNCOV

334



    code = initRowKey(&pScanInfo->sttKeyInfo.nextProcKey, skey, numOfPks, type, bytes, asc);




×








UNCOV

335



    TSDB_CHECK_CODE(code, lino, _end);





×







336



  } else {











UNCOV

337



    int64_t ekey = pReader->info.window.ekey;




×








UNCOV

338



    int64_t ts = (ekey < INT64_MAX) ? (ekey + 1) : ekey;





×







339













UNCOV

340



    code = initRowKey(pRowKey, ts, numOfPks, type, bytes, asc);




×








UNCOV

341



    TSDB_CHECK_CODE(code, lino, _end);





×







342













UNCOV

343



    code = initRowKey(&pScanInfo->sttKeyInfo.nextProcKey, ekey, numOfPks, type, bytes, asc);




×








UNCOV

344



    TSDB_CHECK_CODE(code, lino, _end);





×







345



  }










346













UNCOV

347



  code = initRowKey(&pScanInfo->sttRange.skey, INT64_MAX, numOfPks, type, bytes, asc);




×








UNCOV

348



  TSDB_CHECK_CODE(code, lino, _end);





×







349













UNCOV

350



  code = initRowKey(&pScanInfo->sttRange.ekey, INT64_MIN, numOfPks, type, bytes, asc);




×








UNCOV

351



  TSDB_CHECK_CODE(code, lino, _end);





×







352













UNCOV

353



_end:




×








UNCOV

354



  if (code != TSDB_CODE_SUCCESS) {





×







355



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));





×







356



  }











UNCOV

357



  return code;




×







358



}










359













UNCOV

360



int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSHashObj* pTableMap,




×







361



                               STsdbReader* pReader) {











UNCOV

362



  int32_t code = TSDB_CODE_SUCCESS;




×








UNCOV

363



  int32_t lino = 0;




×







364













UNCOV

365



  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);





×








UNCOV

366



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);





×







367













UNCOV

368



  pScanInfo->uid = uid;




×








UNCOV

369



  INIT_KEYRANGE(&pScanInfo->sttRange);




×








UNCOV

370



  INIT_TIMEWINDOW(&pScanInfo->filesetWindow);




×







371













UNCOV

372



  pScanInfo->cleanSttBlocks = false;




×








UNCOV

373



  pScanInfo->sttBlockReturned = false;




×







374













UNCOV

375



  code = initLastProcKey(pScanInfo, pReader);




×








UNCOV

376



  TSDB_CHECK_CODE(code, lino, _end);





×







377













UNCOV

378



  pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT;




×








UNCOV

379



  code = tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES);




×








UNCOV

380



  TSDB_CHECK_CODE(code, lino, _end);





×







381













UNCOV

382



  tsdbTrace("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pReader, pScanInfo->uid,





×







383



            pScanInfo->lastProcKey.ts, pReader->idStr);










384













UNCOV

385



_end:




×








UNCOV

386



  if (code != TSDB_CODE_SUCCESS) {





×







387



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));





×







388



  }











UNCOV

389



  return code;




×







390



}










391












392



// NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model











UNCOV

393



int32_t createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList,




×







394



                                STableUidList* pUidList, int32_t numOfTables, SSHashObj** pHashObj) {











UNCOV

395



  int32_t    code = TSDB_CODE_SUCCESS;




×








UNCOV

396



  int32_t    lino = 0;




×








UNCOV

397



  SSHashObj* pTableMap = NULL;




×








UNCOV

398



  int64_t    st = 0;




×







399













UNCOV

400



  TSDB_CHECK_NULL(pUidList, code, lino, _end, TSDB_CODE_INVALID_PARA);





×








UNCOV

401



  TSDB_CHECK_NULL(pHashObj, code, lino, _end, TSDB_CODE_INVALID_PARA);





×







402













UNCOV

403



  *pHashObj = NULL;




×







404












405



  // allocate buffer in order to load data blocks from file










406



  // todo use simple hash instead, optimize the memory consumption











UNCOV

407



  pTableMap = tSimpleHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));




×








UNCOV

408



  TSDB_CHECK_NULL(pTableMap, code, lino, _end, terrno);





×







409













UNCOV

410



  st = taosGetTimestampUs();




×








UNCOV

411



  code = initBlockScanInfoBuf(pBuf, numOfTables);




×








UNCOV

412



  if (code != TSDB_CODE_SUCCESS) {





×







413



    tSimpleHashCleanup(pTableMap);




×







414



    TSDB_CHECK_CODE(code, lino, _end);





×







415



  }










416













UNCOV

417



  pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t));





×








UNCOV

418



  if (pUidList->tableUidList == NULL) {





×







419



    tSimpleHashCleanup(pTableMap);




×







420



    TSDB_CHECK_NULL(pUidList->tableUidList, code, lino, _end, terrno);





×







421



  }










422













UNCOV

423



  pUidList->currentIndex = 0;




×







424













UNCOV

425



  for (int32_t j = 0; j < numOfTables; ++j) {





×








UNCOV

426



    pUidList->tableUidList[j] = idList[j].uid;




×







427













UNCOV

428



    STableBlockScanInfo* pScanInfo = NULL;




×








UNCOV

429



    code = getPosInBlockInfoBuf(pBuf, j, &pScanInfo);




×








UNCOV

430



    if (code != TSDB_CODE_SUCCESS) {





×







431



      lino = __LINE__;




×







432



      break;




×







433



    }










434













UNCOV

435



    code = initTableBlockScanInfo(pScanInfo, idList[j].uid, pTableMap, pTsdbReader);




×








UNCOV

436



    if (code != TSDB_CODE_SUCCESS) {





×







437



      lino = __LINE__;




×







438



      break;




×







439



    }










440



  }










441













UNCOV

442



  taosSort(pUidList->tableUidList, numOfTables, sizeof(uint64_t), uidComparFunc);




×







443













UNCOV

444



  pTsdbReader->cost.createScanInfoList = (taosGetTimestampUs() - st) / 1000.0;




×








UNCOV

445



  tsdbDebug("%p create %d tables scan-info, size:%.2f Kb, elapsed time:%.2f ms, %s", pTsdbReader, numOfTables,





×







446



            (sizeof(STableBlockScanInfo) * numOfTables) / 1024.0, pTsdbReader->cost.createScanInfoList,










447



            pTsdbReader->idStr);










448













UNCOV

449



  *pHashObj = pTableMap;




×







450













UNCOV

451



_end:




×








UNCOV

452



  if (code != TSDB_CODE_SUCCESS) {





×







453



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));





×







454



  }











UNCOV

455



  return code;




×







456



}










457













UNCOV

458



void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) {




×








UNCOV

459



  STableBlockScanInfo** p = NULL;




×








UNCOV

460



  int32_t               iter = 0;




×







461













UNCOV

462



  while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) {





×








UNCOV

463



    STableBlockScanInfo* pInfo = *p;




×








UNCOV

464



    if (pInfo == NULL) {





×







465



      continue;




×







466



    }










467













UNCOV

468



    pInfo->iterInit = false;




×








UNCOV

469



    pInfo->iter.hasVal = false;




×








UNCOV

470



    pInfo->iiter.hasVal = false;




×







471













UNCOV

472



    if (pInfo->iter.iter != NULL) {





×








UNCOV

473



      pInfo->iter.iter = tsdbTbDataIterDestroy(pInfo->iter.iter);




×







474



    }










475













UNCOV

476



    if (pInfo->iiter.iter != NULL) {





×








UNCOV

477



      pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter);




×







478



    }










479













UNCOV

480



    taosArrayDestroy(pInfo->delSkyline);




×








UNCOV

481



    pInfo->delSkyline = NULL;




×








UNCOV

482



    pInfo->lastProcKey.ts = ts;




×







483



    // todo check the nextProcKey info











UNCOV

484



    pInfo->sttKeyInfo.nextProcKey.ts = ts + step;




×







485



  }











UNCOV

486



}




×







487













UNCOV

488



void clearBlockScanInfo(STableBlockScanInfo* p) {




×








UNCOV

489



  if (p == NULL) {





×







490



    return;




×







491



  }










492













UNCOV

493



  p->iterInit = false;




×








UNCOV

494



  p->iter.hasVal = false;




×








UNCOV

495



  p->iiter.hasVal = false;




×








UNCOV

496



  p->sttKeyInfo.status = STT_FILE_READER_UNINIT;




×







497













UNCOV

498



  if (p->iter.iter != NULL) {





×








UNCOV

499



    p->iter.iter = tsdbTbDataIterDestroy(p->iter.iter);




×







500



  }










501













UNCOV

502



  if (p->iiter.iter != NULL) {





×








UNCOV

503



    p->iiter.iter = tsdbTbDataIterDestroy(p->iiter.iter);




×







504



  }










505













UNCOV

506



  taosArrayDestroy(p->delSkyline);




×








UNCOV

507



  p->delSkyline = NULL;




×








UNCOV

508



  taosArrayDestroy(p->pBlockList);




×








UNCOV

509



  p->pBlockList = NULL;




×








UNCOV

510



  taosArrayDestroy(p->pBlockIdxList);




×








UNCOV

511



  p->pBlockIdxList = NULL;




×








UNCOV

512



  taosArrayDestroy(p->pMemDelData);




×








UNCOV

513



  p->pMemDelData = NULL;




×








UNCOV

514



  taosArrayDestroy(p->pFileDelData);




×








UNCOV

515



  p->pFileDelData = NULL;




×







516













UNCOV

517



  clearRowKey(&p->lastProcKey);




×








UNCOV

518



  clearRowKey(&p->sttRange.skey);




×








UNCOV

519



  clearRowKey(&p->sttRange.ekey);




×








UNCOV

520



  clearRowKey(&p->sttKeyInfo.nextProcKey);




×







521



}










522













UNCOV

523



void destroyAllBlockScanInfo(SSHashObj** pTableMap) {




×








UNCOV

524



  STableBlockScanInfo** p = NULL;




×








UNCOV

525



  int32_t               iter = 0;




×







526













UNCOV

527



  if (pTableMap == NULL || *pTableMap == NULL) {





×







528



    return;




×







529



  }










530













UNCOV

531



  while ((p = tSimpleHashIterate(*pTableMap, p, &iter)) != NULL) {





×








UNCOV

532



    clearBlockScanInfo(*p);




×







533



  }










534













UNCOV

535



  tSimpleHashCleanup(*pTableMap);




×








UNCOV

536



  *pTableMap = NULL;




×







537



}










538













UNCOV

539



static void doCleanupInfoForNextFileset(STableBlockScanInfo* pScanInfo) {




×








UNCOV

540



  if (pScanInfo == NULL) {





×







541



    return;




×







542



  }










543



  // reset the index in last block when handing a new file











UNCOV

544



  taosArrayClear(pScanInfo->pBlockList);




×








UNCOV

545



  taosArrayClear(pScanInfo->pBlockIdxList);




×








UNCOV

546



  taosArrayClear(pScanInfo->pFileDelData);  // del data from each file set




×








UNCOV

547



  pScanInfo->cleanSttBlocks = false;




×








UNCOV

548



  pScanInfo->numOfRowsInStt = 0;




×








UNCOV

549



  pScanInfo->sttBlockReturned = false;




×








UNCOV

550



  INIT_KEYRANGE(&pScanInfo->sttRange);




×








UNCOV

551



  INIT_TIMEWINDOW(&pScanInfo->filesetWindow);




×








UNCOV

552



  pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT;




×







553



}










554













UNCOV

555



void cleanupInfoForNextFileset(SSHashObj* pTableMap) {




×








UNCOV

556



  STableBlockScanInfo** p = NULL;




×








UNCOV

557



  int32_t               iter = 0;




×







558













UNCOV

559



  while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) {





×








UNCOV

560



    doCleanupInfoForNextFileset(*p);




×







561



  }











UNCOV

562



}




×







563












564



// brin records iterator











UNCOV

565



void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList) {




×








UNCOV

566



  if (pIter == NULL) {





×







567



    return;




×







568



  }










569













UNCOV

570



  (void)memset(&pIter->block, 0, sizeof(SBrinBlock));




×








UNCOV

571



  (void)memset(&pIter->record, 0, sizeof(SBrinRecord));




×








UNCOV

572



  pIter->blockIndex = -1;




×








UNCOV

573



  pIter->recordIndex = -1;




×







574













UNCOV

575



  pIter->pReader = pReader;




×








UNCOV

576



  pIter->pBrinBlockList = pList;




×







577



}










578













UNCOV

579



int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord) {




×








UNCOV

580



  int32_t code = TSDB_CODE_SUCCESS;




×








UNCOV

581



  int32_t lino = 0;




×







582













UNCOV

583



  TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA);





×








UNCOV

584



  TSDB_CHECK_NULL(pRecord, code, lino, _end, TSDB_CODE_INVALID_PARA);





×







585













UNCOV

586



  *pRecord = NULL;




×







587













UNCOV

588



  if (pIter->blockIndex == -1 || (pIter->recordIndex + 1) >= pIter->block.numOfRecords) {





×








UNCOV

589



    pIter->blockIndex += 1;




×








UNCOV

590



    if (pIter->blockIndex >= taosArrayGetSize(pIter->pBrinBlockList)) {





×








UNCOV

591



      goto _end;




×







592



    }










593













UNCOV

594



    pIter->pCurrentBlk = taosArrayGet(pIter->pBrinBlockList, pIter->blockIndex);




×








UNCOV

595



    TSDB_CHECK_NULL(pIter->pCurrentBlk, code, lino, _end, terrno);





×







596













UNCOV

597



    tBrinBlockClear(&pIter->block);




×








UNCOV

598



    TSDB_CHECK_NULL(pIter->pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);





×








UNCOV

599



    code = tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block);




×








UNCOV

600



    if (code != TSDB_CODE_SUCCESS) {





×







601



      tsdbError("failed to read brinBlock from file, code:%s", tstrerror(code));





×








UNCOV

602



      TSDB_CHECK_CODE(code, lino, _end);





×







603



    }










604













UNCOV

605



    pIter->recordIndex = -1;




×







606



  }










607













UNCOV

608



  pIter->recordIndex += 1;




×








UNCOV

609



  code = tBrinBlockGet(&pIter->block, pIter->recordIndex, &pIter->record);




×








UNCOV

610



  *pRecord = &pIter->record;




×








UNCOV

611



  TSDB_CHECK_CODE(code, lino, _end);





×







612













UNCOV

613



_end:




×








UNCOV

614



  if (code != TSDB_CODE_SUCCESS) {





×







615



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));





×







616



  }











UNCOV

617



  return code;




×







618



}










619













UNCOV

620



void clearBrinBlockIter(SBrinRecordIter* pIter) {




×








UNCOV

621



  if (pIter != NULL) {





×








UNCOV

622



    tBrinBlockDestroy(&pIter->block);




×







623



  }











UNCOV

624



}




×







625












626



// initialize the file block access order










627



//  sort the file blocks according to the offset of each data block in the files











UNCOV

628



static void cleanupBlockOrderSupporter(SBlockOrderSupporter* pSup) {




×








UNCOV

629



  if (pSup == NULL) {





×







630



    return;




×







631



  }










632













UNCOV

633



  taosMemoryFreeClear(pSup->numOfBlocksPerTable);





×








UNCOV

634



  taosMemoryFreeClear(pSup->indexPerTable);





×







635













UNCOV

636



  if (pSup->pDataBlockInfo != NULL) {





×








UNCOV

637



    for (int32_t i = 0; i < pSup->numOfTables; ++i) {





×








UNCOV

638



      SBlockOrderWrapper* pBlockInfo = pSup->pDataBlockInfo[i];




×








UNCOV

639



      taosMemoryFreeClear(pBlockInfo);





×







640



    }










641













UNCOV

642



    taosMemoryFreeClear(pSup->pDataBlockInfo);





×







643



  }










644



}










645













UNCOV

646



static int32_t initBlockOrderSupporter(SBlockOrderSupporter* pSup, int32_t numOfTables) {




×








UNCOV

647



  int32_t code = TSDB_CODE_SUCCESS;




×








UNCOV

648



  int32_t lino = 0;




×







649













UNCOV

650



  TSDB_CHECK_NULL(pSup, code, lino, _end, TSDB_CODE_INVALID_PARA);





×








UNCOV

651



  TSDB_CHECK_CONDITION(numOfTables >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA);





×







652













UNCOV

653



  pSup->pDataBlockInfo = taosMemoryCalloc(numOfTables, POINTER_BYTES);





×








UNCOV

654



  TSDB_CHECK_NULL(pSup->pDataBlockInfo, code, lino, _end, terrno);





×








UNCOV

655



  pSup->indexPerTable = taosMemoryCalloc(numOfTables, sizeof(int32_t));





×








UNCOV

656



  TSDB_CHECK_NULL(pSup->indexPerTable, code, lino, _end, terrno);





×








UNCOV

657



  pSup->numOfBlocksPerTable = taosMemoryCalloc(numOfTables, sizeof(int32_t));





×








UNCOV

658



  TSDB_CHECK_NULL(pSup->numOfBlocksPerTable, code, lino, _end, terrno);





×








UNCOV

659



  pSup->numOfTables = 0;




×







660













UNCOV

661



_end:




×








UNCOV

662



  if (code != TSDB_CODE_SUCCESS) {





×







663



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));





×







664



  }











UNCOV

665



  return code;




×







666



}










667













UNCOV

668



static int32_t fileDataBlockOrderCompar(const void* pLeft, const void* pRight, void* param) {




×








UNCOV

669



  int32_t                     leftIndex = 0;




×








UNCOV

670



  int32_t                     rightIndex = 0;




×








UNCOV

671



  int32_t                     leftTableBlockIndex = 0;




×








UNCOV

672



  int32_t                     rightTableBlockIndex = 0;




×








UNCOV

673



  const SBlockOrderSupporter* pSupporter = NULL;




×








UNCOV

674



  const SBlockOrderWrapper*   pLeftBlock = NULL;




×








UNCOV

675



  const SBlockOrderWrapper*   pRightBlock = NULL;




×







676













UNCOV

677



  leftIndex = *(const int32_t*)pLeft;




×








UNCOV

678



  rightIndex = *(const int32_t*)pRight;




×








UNCOV

679



  pSupporter = (const SBlockOrderSupporter*)param;




×







680













UNCOV

681



  leftTableBlockIndex = pSupporter->indexPerTable[leftIndex];




×








UNCOV

682



  rightTableBlockIndex = pSupporter->indexPerTable[rightIndex];




×







683













UNCOV

684



  if (leftTableBlockIndex >= pSupporter->numOfBlocksPerTable[leftIndex]) {





×







685



    /* left block is empty */











UNCOV

686



    return 1;




×








UNCOV

687



  } else if (rightTableBlockIndex >= pSupporter->numOfBlocksPerTable[rightIndex]) {





×







688



    /* right block is empty */











UNCOV

689



    return -1;




×







690



  }










691













UNCOV

692



  pLeftBlock = &pSupporter->pDataBlockInfo[leftIndex][leftTableBlockIndex];




×








UNCOV

693



  pRightBlock = &pSupporter->pDataBlockInfo[rightIndex][rightTableBlockIndex];




×







694













UNCOV

695



  return pLeftBlock->offset > pRightBlock->offset ? 1 : -1;





×







696



}










697













UNCOV

698



int32_t recordToBlockInfo(SFileDataBlockInfo* pBlockInfo, SBrinRecord* record) {




×








UNCOV

699



  int32_t        code = TSDB_CODE_SUCCESS;




×








UNCOV

700



  int32_t        lino = 0;




×








UNCOV

701



  const SRowKey* pFirstKey = NULL;




×








UNCOV

702



  const SRowKey* pLastKey = NULL;




×







703













UNCOV

704



  TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);





×








UNCOV

705



  TSDB_CHECK_NULL(record, code, lino, _end, TSDB_CODE_INVALID_PARA);





×







706













UNCOV

707



  pBlockInfo->uid = record->uid;




×








UNCOV

708



  pBlockInfo->firstKey = record->firstKey.key.ts;




×








UNCOV

709



  pBlockInfo->lastKey = record->lastKey.key.ts;




×








UNCOV

710



  pBlockInfo->minVer = record->minVer;




×








UNCOV

711



  pBlockInfo->maxVer = record->maxVer;




×








UNCOV

712



  pBlockInfo->blockOffset = record->blockOffset;




×








UNCOV

713



  pBlockInfo->smaOffset = record->smaOffset;




×








UNCOV

714



  pBlockInfo->blockSize = record->blockSize;




×








UNCOV

715



  pBlockInfo->blockKeySize = record->blockKeySize;




×








UNCOV

716



  pBlockInfo->smaSize = record->smaSize;




×








UNCOV

717



  pBlockInfo->numRow = record->numRow;




×








UNCOV

718



  pBlockInfo->count = record->count;




×







719













UNCOV

720



  pFirstKey = &record->firstKey.key;




×








UNCOV

721



  pLastKey = &record->lastKey.key;




×








UNCOV

722



  TSDB_CHECK_CONDITION((pFirstKey->numOfPKs == pLastKey->numOfPKs), code, lino, _end, TSDB_CODE_INVALID_PARA);





×








UNCOV

723



  if (pFirstKey->numOfPKs > 0) {





×








UNCOV

724



    if (IS_NUMERIC_TYPE(pFirstKey->pks[0].type)) {





×








UNCOV

725



      pBlockInfo->firstPk.val = pFirstKey->pks[0].val;




×








UNCOV

726



      pBlockInfo->lastPk.val = pLastKey->pks[0].val;




×







727



    } else {










728



      int32_t keyLen = pFirstKey->pks[0].nData;




×







729



      char*   p = taosMemoryMalloc(keyLen + VARSTR_HEADER_SIZE);





×







730



      TSDB_CHECK_NULL(p, code, lino, _end, terrno);





×







731



      memcpy(varDataVal(p), pFirstKey->pks[0].pData, keyLen);




×







732



      varDataSetLen(p, keyLen);




×







733



      pBlockInfo->firstPk.pData = (uint8_t*)p;




×







734












735



      keyLen = pLastKey->pks[0].nData;




×







736



      p = taosMemoryMalloc(keyLen + VARSTR_HEADER_SIZE);





×







737



      TSDB_CHECK_NULL(p, code, lino, _end, terrno);





×







738



      memcpy(varDataVal(p), pLastKey->pks[0].pData, keyLen);




×







739



      varDataSetLen(p, keyLen);




×







740



      pBlockInfo->lastPk.pData = (uint8_t*)p;




×







741



    }










742



  }










743













UNCOV

744



_end:




×








UNCOV

745



  if (code != TSDB_CODE_SUCCESS) {





×







746



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));





×







747



  }











UNCOV

748



  return code;




×







749



}










750












751



static void freePkItem(void* pItem) {




×







752



  SFileDataBlockInfo* p = pItem;




×







753



  if (p != NULL) {





×







754



    taosMemoryFreeClear(p->firstPk.pData);





×







755



    taosMemoryFreeClear(p->lastPk.pData);





×







756



  }










757



}




×







758













UNCOV

759



void clearDataBlockIterator(SDataBlockIter* pIter, bool needFree) {




×








UNCOV

760



  if (pIter == NULL) {





×







761



    return;




×







762



  }










763













UNCOV

764



  pIter->index = -1;




×








UNCOV

765



  pIter->numOfBlocks = 0;




×







766













UNCOV

767



  if (needFree) {





×








UNCOV

768



    taosArrayClearEx(pIter->blockList, freePkItem);




×







769



  } else {











UNCOV

770



    taosArrayClear(pIter->blockList);




×







771



  }










772



}










773













UNCOV

774



void cleanupDataBlockIterator(SDataBlockIter* pIter, bool needFree) {




×








UNCOV

775



  if (pIter == NULL) {





×







776



    return;




×







777



  }










778













UNCOV

779



  pIter->index = -1;




×








UNCOV

780



  pIter->numOfBlocks = 0;




×








UNCOV

781



  if (needFree) {





×








UNCOV

782



    taosArrayDestroyEx(pIter->blockList, freePkItem);




×







783



  } else {











UNCOV

784



    taosArrayDestroy(pIter->blockList);




×







785



  }











UNCOV

786



  pIter->blockList = NULL;




×







787



}










788













UNCOV

789



int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList) {




×








UNCOV

790



  int32_t                   code = TSDB_CODE_SUCCESS;




×








UNCOV

791



  int32_t                   lino = 0;




×








UNCOV

792



  bool                      asc = false;




×








UNCOV

793



  int32_t                   numOfTables = 0;




×








UNCOV

794



  int64_t                   st = 0;




×








UNCOV

795



  int64_t                   et = 0;




×








UNCOV

796



  int32_t                   cnt = 0;




×








UNCOV

797



  SBlockOrderSupporter      sup = {0};




×








UNCOV

798



  SMultiwayMergeTreeInfo*   pTree = NULL;




×








UNCOV

799



  STableBlockScanInfo*      pTableScanInfo = NULL;




×








UNCOV

800



  const SFileDataBlockInfo* pBlockInfo = NULL;




×








UNCOV

801



  const void*               px = NULL;




×







802













UNCOV

803



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);





×








UNCOV

804



  TSDB_CHECK_NULL(pBlockIter, code, lino, _end, TSDB_CODE_INVALID_PARA);





×








UNCOV

805



  TSDB_CHECK_CONDITION(numOfBlocks >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA);





×







806













UNCOV

807



  asc = ASCENDING_TRAVERSE(pReader->info.order);




×








UNCOV

808



  clearDataBlockIterator(pBlockIter, shouldFreePkBuf(&pReader->suppInfo));




×








UNCOV

809



  pBlockIter->numOfBlocks = numOfBlocks;




×







810












811



  // access data blocks according to the offset of each block in asc/desc order.











UNCOV

812



  numOfTables = taosArrayGetSize(pTableList);




×







813













UNCOV

814



  st = taosGetTimestampUs();




×








UNCOV

815



  code = initBlockOrderSupporter(&sup, numOfTables);




×








UNCOV

816



  TSDB_CHECK_CODE(code, lino, _end);





×







817













UNCOV

818



  for (int32_t i = 0; i < numOfTables; ++i) {





×








UNCOV

819



    pTableScanInfo = taosArrayGetP(pTableList, i);




×







820













UNCOV

821



    size_t              num = taosArrayGetSize(pTableScanInfo->pBlockList);




×








UNCOV

822



    SBlockOrderWrapper* buf = taosMemoryMalloc(sizeof(SBlockOrderWrapper) * num);





×








UNCOV

823



    TSDB_CHECK_NULL(buf, code, lino, _end, terrno);





×








UNCOV

824



    sup.numOfBlocksPerTable[sup.numOfTables] = num;




×








UNCOV

825



    sup.pDataBlockInfo[sup.numOfTables] = buf;




×








UNCOV

826



    sup.numOfTables++;




×







827













UNCOV

828



    for (int32_t k = 0; k < num; ++k) {





×








UNCOV

829



      pBlockInfo = taosArrayGet(pTableScanInfo->pBlockList, k);




×








UNCOV

830



      TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);





×







831













UNCOV

832



      sup.pDataBlockInfo[i][k] =




×








UNCOV

833



          (SBlockOrderWrapper){.uid = pTableScanInfo->uid, .offset = pBlockInfo->blockOffset, .pInfo = pTableScanInfo};




×








UNCOV

834



      cnt++;




×







835



    }










836



  }










837













UNCOV

838



  TSDB_CHECK_CONDITION(!(numOfBlocks != cnt && sup.numOfTables != numOfTables), code, lino, _end,





×







839



                       TSDB_CODE_INVALID_PARA);










840












841



  // since there is only one table qualified, blocks are not sorted











UNCOV

842



  if (sup.numOfTables == 1) {





×








UNCOV

843



    pTableScanInfo = taosArrayGetP(pTableList, 0);




×








UNCOV

844



    for (int32_t i = 0; i < numOfBlocks; ++i) {





×








UNCOV

845



      STableDataBlockIdx tableDataBlockIdx = {.globalIndex = i};




×








UNCOV

846



      px = taosArrayPush(pTableScanInfo->pBlockIdxList, &tableDataBlockIdx);




×








UNCOV

847



      TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







848



    }










849













UNCOV

850



    px = taosArrayAddAll(pBlockIter->blockList, pTableScanInfo->pBlockList);




×








UNCOV

851



    TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







852













UNCOV

853



    taosArrayDestroy(pTableScanInfo->pBlockList);




×








UNCOV

854



    pTableScanInfo->pBlockList = NULL;




×







855













UNCOV

856



    et = taosGetTimestampUs();




×








UNCOV

857



    tsdbDebug("%p create blocks info struct completed for one table, %d blocks not sorted, elapsed time:%.2f ms %s",





×







858



              pReader, numOfBlocks, (et - st) / 1000.0, pReader->idStr);










859













UNCOV

860



    pBlockIter->index = asc ? 0 : (numOfBlocks - 1);





×








UNCOV

861



    goto _end;




×







862



  }










863













UNCOV

864



  tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables %s", pReader, cnt, sup.numOfTables,





×







865



            pReader->idStr);










866













UNCOV

867



  code = tMergeTreeCreate(&pTree, sup.numOfTables, &sup, fileDataBlockOrderCompar);




×








UNCOV

868



  TSDB_CHECK_CODE(code, lino, _end);





×







869













UNCOV

870



  for (int32_t i = 0; i < cnt; ++i) {





×








UNCOV

871



    int32_t pos = tMergeTreeGetChosenIndex(pTree);




×








UNCOV

872



    int32_t index = sup.indexPerTable[pos]++;




×








UNCOV

873



    pTableScanInfo = sup.pDataBlockInfo[pos][index].pInfo;




×







874













UNCOV

875



    pBlockInfo = taosArrayGet(pTableScanInfo->pBlockList, index);




×








UNCOV

876



    TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, terrno);





×







877













UNCOV

878



    px = taosArrayPush(pBlockIter->blockList, pBlockInfo);




×








UNCOV

879



    TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







880













UNCOV

881



    STableDataBlockIdx tableDataBlockIdx = {.globalIndex = i};




×








UNCOV

882



    px = taosArrayPush(pTableScanInfo->pBlockIdxList, &tableDataBlockIdx);




×








UNCOV

883



    TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







884












885



    // set data block index overflow, in order to disable the offset comparator











UNCOV

886



    if (sup.indexPerTable[pos] >= sup.numOfBlocksPerTable[pos]) {





×








UNCOV

887



      sup.indexPerTable[pos] = sup.numOfBlocksPerTable[pos] + 1;




×







888



    }










889













UNCOV

890



    code = tMergeTreeAdjust(pTree, tMergeTreeGetAdjustIndex(pTree));




×








UNCOV

891



    TSDB_CHECK_CODE(code, lino, _end);





×







892



  }










893













UNCOV

894



  for (int32_t i = 0; i < numOfTables; ++i) {





×








UNCOV

895



    pTableScanInfo = taosArrayGetP(pTableList, i);




×








UNCOV

896



    taosArrayDestroy(pTableScanInfo->pBlockList);




×








UNCOV

897



    pTableScanInfo->pBlockList = NULL;




×







898



  }










899













UNCOV

900



  et = taosGetTimestampUs();




×








UNCOV

901



  tsdbDebug("%p %d data blocks access order completed, elapsed time:%.2f ms %s", pReader, numOfBlocks,





×







902



            (et - st) / 1000.0, pReader->idStr);










903













UNCOV

904



  pBlockIter->index = asc ? 0 : (numOfBlocks - 1);





×







905













UNCOV

906



_end:




×








UNCOV

907



  if (code != TSDB_CODE_SUCCESS) {





×







908



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));





×







909



  }











UNCOV

910



  cleanupBlockOrderSupporter(&sup);




×








UNCOV

911



  if (pTree != NULL) {





×








UNCOV

912



    tMergeTreeDestroy(&pTree);




×







913



  }











UNCOV

914



  return code;




×







915



}










916













UNCOV

917



bool blockIteratorNext(SDataBlockIter* pBlockIter) {




×








UNCOV

918



  bool asc = false;




×







919













UNCOV

920



  if (pBlockIter == NULL) {





×







921



    return false;




×







922



  }










923













UNCOV

924



  asc = ASCENDING_TRAVERSE(pBlockIter->order);




×








UNCOV

925



  if ((pBlockIter->index >= pBlockIter->numOfBlocks - 1 && asc) || (pBlockIter->index <= 0 && (!asc))) {





×








UNCOV

926



    return false;




×







927



  }











UNCOV

928



  pBlockIter->index += asc ? 1 : -1;





×








UNCOV

929



  return true;




×







930



}










931












932



typedef enum {










933



  BLK_CHECK_CONTINUE = 0x1,










934



  BLK_CHECK_QUIT = 0x2,










935



} ETombBlkCheckEnum;










936












937



static int32_t loadNextStatisticsBlock(SSttFileReader* pSttFileReader, STbStatisBlock* pStatisBlock,










938



                                       const TStatisBlkArray* pStatisBlkArray, int32_t numOfRows, int32_t* i,










939



                                       int32_t* j);











UNCOV

940



static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_t numOfTables, int32_t* j,




×







941



                                ETombBlkCheckEnum* pRet) {











UNCOV

942



  int32_t              code = TSDB_CODE_SUCCESS;




×








UNCOV

943



  int32_t              lino = 0;




×








UNCOV

944



  STombRecord          record = {0};




×








UNCOV

945



  uint64_t             uid = 0;




×








UNCOV

946



  STableBlockScanInfo* pScanInfo = NULL;




×







947













UNCOV

948



  TSDB_CHECK_NULL(pBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);





×








UNCOV

949



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);





×








UNCOV

950



  TSDB_CHECK_NULL(pRet, code, lino, _end, TSDB_CODE_INVALID_PARA);





×







951













UNCOV

952



  *pRet = BLK_CHECK_QUIT;




×








UNCOV

953



  uid = pReader->status.uidList.tableUidList[*j];




×








UNCOV

954



  code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr);




×








UNCOV

955



  TSDB_CHECK_CODE(code, lino, _end);





×







956













UNCOV

957



  if (pScanInfo->pFileDelData == NULL) {





×








UNCOV

958



    pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData));




×








UNCOV

959



    TSDB_CHECK_NULL(pScanInfo->pFileDelData, code, lino, _end, terrno);





×







960



  }










961













UNCOV

962



  for (int32_t k = 0; k < pBlock->numOfRecords; ++k) {





×








UNCOV

963



    code = tTombBlockGet(pBlock, k, &record);




×








UNCOV

964



    TSDB_CHECK_CODE(code, lino, _end);





×







965













UNCOV

966



    if (record.suid < pReader->info.suid) {





×








UNCOV

967



      continue;




×







968



    }










969













UNCOV

970



    if (record.suid > pReader->info.suid) {





×








UNCOV

971



      *pRet = BLK_CHECK_QUIT;




×








UNCOV

972



      goto _end;




×







973



    }










974













UNCOV

975



    if (uid < record.uid) {





×








UNCOV

976



      while ((*j) < numOfTables && pReader->status.uidList.tableUidList[*j] < record.uid) {





×








UNCOV

977



        (*j) += 1;




×







978



      }










979













UNCOV

980



      if ((*j) >= numOfTables) {





×








UNCOV

981



        *pRet = BLK_CHECK_QUIT;




×








UNCOV

982



        goto _end;




×







983



      }










984













UNCOV

985



      uid = pReader->status.uidList.tableUidList[*j];




×








UNCOV

986



      code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr);




×








UNCOV

987



      TSDB_CHECK_CODE(code, lino, _end);





×







988













UNCOV

989



      if (pScanInfo->pFileDelData == NULL) {





×








UNCOV

990



        pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData));




×








UNCOV

991



        TSDB_CHECK_NULL(pScanInfo->pFileDelData, code, lino, _end, terrno);





×







992



      }










993



    }










994













UNCOV

995



    if (record.uid < uid) {





×








UNCOV

996



      continue;




×







997



    }










998













UNCOV

999



    TSDB_CHECK_CONDITION((record.suid == pReader->info.suid) && (uid == record.uid), code, lino, _end,





×







1000



                         TSDB_CODE_INTERNAL_ERROR);










1001













UNCOV

1002



    if (record.version <= pReader->info.verRange.maxVer) {





×








UNCOV

1003



      SDelData    delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey};




×








UNCOV

1004



      const void* px = taosArrayPush(pScanInfo->pFileDelData, &delData);




×








UNCOV

1005



      TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







1006



    }










1007



  }










1008













UNCOV

1009



  *pRet = BLK_CHECK_CONTINUE;




×







1010













UNCOV

1011



_end:




×








UNCOV

1012



  if (code != TSDB_CODE_SUCCESS) {





×







1013



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));





×







1014



  }











UNCOV

1015



  return code;




×







1016



}










1017












1018



// load tomb data API











UNCOV

1019



static int32_t doLoadTombDataFromTombBlk(const TTombBlkArray* pTombBlkArray, STsdbReader* pReader, void* pFileReader,




×







1020



                                         bool isFile) {











UNCOV

1021



  int32_t              code = TSDB_CODE_SUCCESS;




×








UNCOV

1022



  int32_t              lino = 0;




×








UNCOV

1023



  const STableUidList* pList = NULL;




×








UNCOV

1024



  int32_t              numOfTables = 0;




×







1025













UNCOV

1026



  TSDB_CHECK_NULL(pTombBlkArray, code, lino, _end, TSDB_CODE_INVALID_PARA);





×








UNCOV

1027



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);





×







1028













UNCOV

1029



  pList = &pReader->status.uidList;




×








UNCOV

1030



  numOfTables = tSimpleHashGetSize(pReader->status.pTableMap);




×







1031













UNCOV

1032



  int32_t i = 0, j = 0;




×








UNCOV

1033



  while (i < pTombBlkArray->size && j < numOfTables) {





×








UNCOV

1034



    STombBlk* pTombBlk = &pTombBlkArray->data[i];




×








UNCOV

1035



    if (pTombBlk->maxTbid.suid < pReader->info.suid) {





×








UNCOV

1036



      i += 1;




×








UNCOV

1037



      continue;




×







1038



    }










1039













UNCOV

1040



    if (pTombBlk->minTbid.suid > pReader->info.suid) {





×








UNCOV

1041



      break;




×







1042



    }










1043













UNCOV

1044



    TSDB_CHECK_CONDITION(





×







1045



        (pTombBlk->minTbid.suid <= pReader->info.suid) && (pTombBlk->maxTbid.suid >= pReader->info.suid), code, lino,










1046



        _end, TSDB_CODE_INTERNAL_ERROR);










1047













UNCOV

1048



    if (pTombBlk->maxTbid.suid == pReader->info.suid && pTombBlk->maxTbid.uid < pList->tableUidList[0]) {





×








UNCOV

1049



      i += 1;




×








UNCOV

1050



      continue;




×







1051



    }










1052













UNCOV

1053



    if (pTombBlk->minTbid.suid == pReader->info.suid && pTombBlk->minTbid.uid > pList->tableUidList[numOfTables - 1]) {





×








UNCOV

1054



      break;




×







1055



    }










1056













UNCOV

1057



    STombBlock block = {0};




×








UNCOV

1058



    code = isFile ? tsdbDataFileReadTombBlock(pFileReader, &pTombBlkArray->data[i], &block)




×








UNCOV

1059



                  : tsdbSttFileReadTombBlock(pFileReader, &pTombBlkArray->data[i], &block);





×








UNCOV

1060



    TSDB_CHECK_CODE(code, lino, _end);





×







1061













UNCOV

1062



    ETombBlkCheckEnum ret = 0;




×








UNCOV

1063



    code = doCheckTombBlock(&block, pReader, numOfTables, &j, &ret);




×







1064













UNCOV

1065



    tTombBlockDestroy(&block);




×








UNCOV

1066



    TSDB_CHECK_CODE(code, lino, _end);





×








UNCOV

1067



    if (ret == BLK_CHECK_QUIT) {





×








UNCOV

1068



      break;




×







1069



    }










1070













UNCOV

1071



    i += 1;




×







1072



  }










1073













UNCOV

1074



_end:




×








UNCOV

1075



  if (code != TSDB_CODE_SUCCESS) {





×







1076



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));





×







1077



  }











UNCOV

1078



  return code;




×







1079



}










1080













UNCOV

1081



int32_t loadDataFileTombDataForAll(STsdbReader* pReader) {




×








UNCOV

1082



  int32_t              code = TSDB_CODE_SUCCESS;




×








UNCOV

1083



  int32_t              lino = 0;




×








UNCOV

1084



  const TTombBlkArray* pBlkArray = NULL;




×







1085













UNCOV

1086



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);





×







1087













UNCOV

1088



  if ((pReader->status.pCurrentFileset == NULL) || (pReader->status.pCurrentFileset->farr[TSDB_FTYPE_TOMB] == NULL)) {





×








UNCOV

1089



    return TSDB_CODE_SUCCESS;




×







1090



  }










1091













UNCOV

1092



  code = tsdbDataFileReadTombBlk(pReader->pFileReader, &pBlkArray);




×








UNCOV

1093



  TSDB_CHECK_CODE(code, lino, _end);





×







1094













UNCOV

1095



  code = doLoadTombDataFromTombBlk(pBlkArray, pReader, pReader->pFileReader, true);




×








UNCOV

1096



  TSDB_CHECK_CODE(code, lino, _end);





×







1097













UNCOV

1098



_end:




×








UNCOV

1099



  if (code != TSDB_CODE_SUCCESS) {





×







1100



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));





×







1101



  }











UNCOV

1102



  return code;




×







1103



}










1104













UNCOV

1105



int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pLoadInfo) {




×








UNCOV

1106



  int32_t              code = TSDB_CODE_SUCCESS;




×








UNCOV

1107



  int32_t              lino = 0;




×








UNCOV

1108



  const TTombBlkArray* pBlkArray = NULL;




×







1109













UNCOV

1110



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);





×







1111













UNCOV

1112



  code = tsdbSttFileReadTombBlk(pSttFileReader, &pBlkArray);




×








UNCOV

1113



  TSDB_CHECK_CODE(code, lino, _end);





×







1114













UNCOV

1115



  code = doLoadTombDataFromTombBlk(pBlkArray, pReader, pSttFileReader, false);




×








UNCOV

1116



  TSDB_CHECK_CODE(code, lino, _end);





×







1117













UNCOV

1118



_end:




×








UNCOV

1119



  if (code != TSDB_CODE_SUCCESS) {





×







1120



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));





×







1121



  }











UNCOV

1122



  return code;




×







1123



}










1124













UNCOV

1125



int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver) {




×








UNCOV

1126



  int32_t         code = TSDB_CODE_SUCCESS;




×








UNCOV

1127



  int32_t         lino = 0;




×








UNCOV

1128



  SArray*         pMemDelData = NULL;




×








UNCOV

1129



  const SDelData* p = NULL;




×








UNCOV

1130



  const void*     px = NULL;




×







1131













UNCOV

1132



  TSDB_CHECK_NULL(ppMemDelData, code, lino, _end, TSDB_CODE_INVALID_PARA);





×







1133













UNCOV

1134



  if (*ppMemDelData == NULL) {





×








UNCOV

1135



    *ppMemDelData = taosArrayInit(4, sizeof(SDelData));




×








UNCOV

1136



    TSDB_CHECK_NULL(*ppMemDelData, code, lino, _end, terrno);





×







1137



  }










1138













UNCOV

1139



  pMemDelData = *ppMemDelData;




×







1140













UNCOV

1141



  if (pMemTbData != NULL) {





×








UNCOV

1142



    taosRLockLatch(&pMemTbData->lock);




×








UNCOV

1143



    p = pMemTbData->pHead;




×








UNCOV

1144



    while (p) {





×








UNCOV

1145



      if (p->version <= ver) {





×








UNCOV

1146



        px = taosArrayPush(pMemDelData, p);




×








UNCOV

1147



        if (px == NULL) {





×







1148



          taosRUnLockLatch(&pMemTbData->lock);




×







1149



          TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







1150



        }










1151



      }











UNCOV

1152



      p = p->pNext;




×







1153



    }











UNCOV

1154



    taosRUnLockLatch(&pMemTbData->lock);




×







1155



  }










1156













UNCOV

1157



  if (piMemTbData != NULL) {





×








UNCOV

1158



    p = piMemTbData->pHead;




×








UNCOV

1159



    while (p) {





×








UNCOV

1160



      if (p->version <= ver) {





×








UNCOV

1161



        px = taosArrayPush(pMemDelData, p);




×








UNCOV

1162



        TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







1163



      }











UNCOV

1164



      p = p->pNext;




×







1165



    }










1166



  }










1167













UNCOV

1168



_end:




×








UNCOV

1169



  if (code != TSDB_CODE_SUCCESS) {





×







1170



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));





×







1171



  }











UNCOV

1172



  return code;




×







1173



}










1174













UNCOV

1175



int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo,




×







1176



                               TStatisBlkArray* pStatisBlkArray, uint64_t suid, const uint64_t* pUidList,










1177



                               int32_t numOfTables, int32_t* pNumOfRows) {











UNCOV

1178



  int32_t           code = TSDB_CODE_SUCCESS;




×








UNCOV

1179



  int32_t           lino = 0;




×








UNCOV

1180



  int32_t           num = 0;




×








UNCOV

1181



  int64_t           st = 0;




×








UNCOV

1182



  const SStatisBlk* p = NULL;




×








UNCOV

1183



  STbStatisBlock*   pStatisBlock = NULL;




×







1184













UNCOV

1185



  TSDB_CHECK_NULL(pSttFileReader, code, lino, _end, TSDB_CODE_INVALID_PARA);





×








UNCOV

1186



  TSDB_CHECK_NULL(pBlockLoadInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);





×








UNCOV

1187



  TSDB_CHECK_NULL(pStatisBlkArray, code, lino, _end, TSDB_CODE_INVALID_PARA);





×








UNCOV

1188



  TSDB_CHECK_NULL(pNumOfRows, code, lino, _end, TSDB_CODE_INVALID_PARA);





×







1189













UNCOV

1190



  *pNumOfRows = 0;




×







1191













UNCOV

1192



  if (TARRAY2_SIZE(pStatisBlkArray) <= 0) {





×







1193



    return code;




×







1194



  }










1195













UNCOV

1196



  int32_t i = 0;




×








UNCOV

1197



  while ((i < TARRAY2_SIZE(pStatisBlkArray)) && (pStatisBlkArray->data[i].maxTbid.suid < suid)) {





×








UNCOV

1198



    ++i;




×







1199



  }










1200













UNCOV

1201



  if (i >= TARRAY2_SIZE(pStatisBlkArray)) {





×








UNCOV

1202



    goto _end;




×







1203



  }










1204













UNCOV

1205



  p = &pStatisBlkArray->data[i];




×








UNCOV

1206



  pStatisBlock = taosMemoryCalloc(1, sizeof(STbStatisBlock));





×








UNCOV

1207



  TSDB_CHECK_NULL(pStatisBlock, code, lino, _end, terrno);





×







1208













UNCOV

1209



  code = tStatisBlockInit(pStatisBlock);




×








UNCOV

1210



  TSDB_CHECK_CODE(code, lino, _end);





×







1211













UNCOV

1212



  st = taosGetTimestampUs();




×








UNCOV

1213



  code = tsdbSttFileReadStatisBlock(pSttFileReader, p, pStatisBlock);




×








UNCOV

1214



  TSDB_CHECK_CODE(code, lino, _end);





×







1215













UNCOV

1216



  double el = (taosGetTimestampUs() - st) / 1000.0;




×








UNCOV

1217



  pBlockLoadInfo->cost.loadStatisBlocks += 1;




×








UNCOV

1218



  pBlockLoadInfo->cost.statisElapsedTime += el;




×







1219













UNCOV

1220



  int32_t index = 0;




×








UNCOV

1221



  while (index < pStatisBlock->numOfRecords && ((int64_t*)pStatisBlock->suids.data)[index] < suid) {





×








UNCOV

1222



    ++index;




×







1223



  }










1224













UNCOV

1225



  if (index >= pStatisBlock->numOfRecords) {





×







1226



    *pNumOfRows = num;




×







1227



    goto _end;




×







1228



  }










1229













UNCOV

1230



  int32_t j = index;




×








UNCOV

1231



  int32_t uidIndex = 0;




×








UNCOV

1232



  while (i < TARRAY2_SIZE(pStatisBlkArray) && uidIndex < numOfTables) {





×








UNCOV

1233



    p = &pStatisBlkArray->data[i];




×








UNCOV

1234



    if (p->minTbid.suid > suid) {





×








UNCOV

1235



      *pNumOfRows = num;




×








UNCOV

1236



      goto _end;




×







1237



    }










1238













UNCOV

1239



    uint64_t uid = pUidList[uidIndex];




×







1240













UNCOV

1241



    if (((int64_t*)pStatisBlock->uids.data)[j] == uid) {





×








UNCOV

1242



      num += ((int64_t*)pStatisBlock->counts.data)[j];




×








UNCOV

1243



      uidIndex += 1;




×








UNCOV

1244



      j += 1;




×








UNCOV

1245



      code = loadNextStatisticsBlock(pSttFileReader, pStatisBlock, pStatisBlkArray, pStatisBlock->numOfRecords, &i, &j);




×








UNCOV

1246



      TSDB_CHECK_CODE(code, lino, _end);





×








UNCOV

1247



    } else if (((int64_t*)pStatisBlock->uids.data)[j] < uid) {





×







1248



      j += 1;




×







1249



      code = loadNextStatisticsBlock(pSttFileReader, pStatisBlock, pStatisBlkArray, pStatisBlock->numOfRecords, &i, &j);




×







1250



      TSDB_CHECK_CODE(code, lino, _end);





×







1251



    } else {











UNCOV

1252



      uidIndex += 1;




×







1253



    }










1254



  }










1255













UNCOV

1256



  *pNumOfRows = num;




×







1257













UNCOV

1258



_end:




×








UNCOV

1259



  if (code != TSDB_CODE_SUCCESS) {





×







1260



    tsdbError("%s with %p failed at line %d since %s", __func__, pSttFileReader, lino, tstrerror(code));





×







1261



  }











UNCOV

1262



  if (pStatisBlock) {





×








UNCOV

1263



    tStatisBlockDestroy(pStatisBlock);




×








UNCOV

1264



    taosMemoryFreeClear(pStatisBlock);





×







1265



  }











UNCOV

1266



  return code;




×







1267



}










1268












1269



// load next stt statistics block











UNCOV

1270



static int32_t loadNextStatisticsBlock(SSttFileReader* pSttFileReader, STbStatisBlock* pStatisBlock,




×







1271



                                       const TStatisBlkArray* pStatisBlkArray, int32_t numOfRows, int32_t* i,










1272



                                       int32_t* j) {











UNCOV

1273



  int32_t code = TSDB_CODE_SUCCESS;




×








UNCOV

1274



  int32_t lino = 0;




×







1275













UNCOV

1276



  if ((*j) >= numOfRows) {





×








UNCOV

1277



    (*i) += 1;




×








UNCOV

1278



    (*j) = 0;




×








UNCOV

1279



    if ((*i) < TARRAY2_SIZE(pStatisBlkArray)) {





×







1280



      code = tsdbSttFileReadStatisBlock(pSttFileReader, &pStatisBlkArray->data[(*i)], pStatisBlock);




×







1281



      if (code != 0) {





×







1282



        tsdbError("%p failed to read statisBlock, code:%s", pSttFileReader, tstrerror(code));





×







1283



        TSDB_CHECK_CODE(code, lino, _end);





×







1284



      }










1285



    }










1286



  }










1287













UNCOV

1288



_end:




×








UNCOV

1289



  if (code != TSDB_CODE_SUCCESS) {





×







1290



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));





×







1291



  }











UNCOV

1292



  return code;




×







1293



}










1294













UNCOV

1295



int32_t doAdjustValidDataIters(SArray* pLDIterList, int32_t numOfFileObj) {




×








UNCOV

1296



  int32_t     code = TSDB_CODE_SUCCESS;




×








UNCOV

1297



  int32_t     lino = 0;




×








UNCOV

1298



  int32_t     size = 0;




×








UNCOV

1299



  int32_t     inc = 0;




×








UNCOV

1300



  SLDataIter* pIter = NULL;




×







1301













UNCOV

1302



  size = taosArrayGetSize(pLDIterList);




×







1303













UNCOV

1304



  if (size < numOfFileObj) {





×








UNCOV

1305



    inc = numOfFileObj - size;




×








UNCOV

1306



    for (int32_t k = 0; k < inc; ++k) {





×








UNCOV

1307



      pIter = taosMemoryCalloc(1, sizeof(SLDataIter));





×








UNCOV

1308



      TSDB_CHECK_NULL(pIter, code, lino, _end, terrno);





×








UNCOV

1309



      void* px = taosArrayPush(pLDIterList, &pIter);




×








UNCOV

1310



      if (px == NULL) {





×







1311



        TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







1312



        taosMemoryFreeClear(pIter);





×







1313



      }










1314



    }











UNCOV

1315



  } else if (size > numOfFileObj) {  // remove unused LDataIter





×







1316



    inc = size - numOfFileObj;




×







1317












1318



    for (int32_t i = 0; i < inc; ++i) {





×







1319



      pIter = taosArrayPop(pLDIterList);




×







1320



      destroyLDataIter(pIter);




×







1321



    }










1322



  }










1323













UNCOV

1324



_end:




×








UNCOV

1325



  if (code != TSDB_CODE_SUCCESS) {





×







1326



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));





×







1327



  }











UNCOV

1328



  return code;




×







1329



}










1330













UNCOV

1331



int32_t adjustSttDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet) {




×








UNCOV

1332



  int32_t  code = TSDB_CODE_SUCCESS;




×








UNCOV

1333



  int32_t  lino = 0;




×








UNCOV

1334



  int32_t  numOfLevels = 0;




×








UNCOV

1335



  SSttLvl* pSttLevel = NULL;




×








UNCOV

1336



  SArray*  pList = NULL;




×







1337













UNCOV

1338



  TSDB_CHECK_NULL(pFileSet, code, lino, _end, TSDB_CODE_INVALID_PARA);





×







1339













UNCOV

1340



  numOfLevels = pFileSet->lvlArr->size;




×







1341












1342



  // add the list/iter placeholder











UNCOV

1343



  while (taosArrayGetSize(pSttFileBlockIterArray) < numOfLevels) {





×








UNCOV

1344



    pList = taosArrayInit(4, POINTER_BYTES);




×








UNCOV

1345



    TSDB_CHECK_NULL(pList, code, lino, _end, terrno);





×








UNCOV

1346



    void* px = taosArrayPush(pSttFileBlockIterArray, &pList);




×








UNCOV

1347



    if (px == NULL) {





×







1348



      taosArrayDestroy(pList);




×







1349



      TSDB_CHECK_NULL(px, code, lino, _end, terrno);





×







1350



    }










1351



  }










1352













UNCOV

1353



  for (int32_t j = 0; j < numOfLevels; ++j) {





×








UNCOV

1354



    pSttLevel = pFileSet->lvlArr->data[j];




×








UNCOV

1355



    pList = taosArrayGetP(pSttFileBlockIterArray, j);




×








UNCOV

1356



    code = doAdjustValidDataIters(pList, TARRAY2_SIZE(pSttLevel->fobjArr));




×








UNCOV

1357



    TSDB_CHECK_CODE(code, lino, _end);





×







1358



  }










1359













UNCOV

1360



_end:




×








UNCOV

1361



  if (code != TSDB_CODE_SUCCESS) {





×







1362



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));





×







1363



  }











UNCOV

1364



  return code;




×







1365



}










1366













UNCOV

1367



int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArray, STsdb* pTsdb, SMergeTreeConf* pConf,




×







1368



                              const char* pstr) {











UNCOV

1369



  int32_t code = TSDB_CODE_SUCCESS;




×








UNCOV

1370



  int32_t lino = 0;




×








UNCOV

1371



  int32_t numOfRows = 0;




×








UNCOV

1372



  int32_t numOfLevels = 0;




×







1373













UNCOV

1374



  TSDB_CHECK_NULL(pFileSet, code, lino, _end, TSDB_CODE_INVALID_PARA);





×







1375












1376



  // no data exists, go to end











UNCOV

1377



  numOfLevels = pFileSet->lvlArr->size;




×








UNCOV

1378



  if (numOfLevels == 0) {





×








UNCOV

1379



    goto _end;




×







1380



  }










1381












1382



  // add the list/iter placeholder











UNCOV

1383



  code = adjustSttDataIters(pSttFileBlockIterArray, pFileSet);




×








UNCOV

1384



  TSDB_CHECK_CODE(code, lino, _end);





×







1385













UNCOV

1386



  for (int32_t j = 0; j < numOfLevels; ++j) {





×








UNCOV

1387



    SSttLvl* pSttLevel = pFileSet->lvlArr->data[j];




×








UNCOV

1388



    SArray*  pList = taosArrayGetP(pSttFileBlockIterArray, j);




×







1389













UNCOV

1390



    for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) {  // open all last file





×








UNCOV

1391



      SLDataIter* pIter = taosArrayGetP(pList, i);




×







1392












1393



      // open stt file reader if not opened yet










1394



      // if failed to open this stt file, ignore the error and try next one











UNCOV

1395



      if (pIter->pReader == NULL) {





×








UNCOV

1396



        SSttFileReaderConfig conf = {.tsdb = pTsdb, .szPage = pTsdb->pVnode->config.tsdbPageSize};




×








UNCOV

1397



        conf.file[0] = *pSttLevel->fobjArr->data[i]->f;




×







1398













UNCOV

1399



        const char* pName = pSttLevel->fobjArr->data[i]->fname;




×








UNCOV

1400



        code = tsdbSttFileReaderOpen(pName, &conf, &pIter->pReader);




×








UNCOV

1401



        if (code != TSDB_CODE_SUCCESS) {





×







1402



          tsdbError("open stt file reader error. file:%s, code %s, %s", pName, tstrerror(code), pstr);





×







1403



          continue;




×







1404



        }










1405



      }










1406













UNCOV

1407



      if (pIter->pBlockLoadInfo == NULL) {





×








UNCOV

1408



        code = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols, &pIter->pBlockLoadInfo);




×








UNCOV

1409



        if (code != TSDB_CODE_SUCCESS) {





×







1410



          tsdbError("failed to create block load info, code: out of memory, %s", pstr);





×







1411



          continue;




×







1412



        }










1413



      }










1414












1415



      // load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file











UNCOV

1416



      TStatisBlkArray* pStatisBlkArray = NULL;




×








UNCOV

1417



      code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray**)&pStatisBlkArray);




×








UNCOV

1418



      if (code != TSDB_CODE_SUCCESS) {





×







1419



        tsdbError("failed to load stt block statistics, code:%s, %s", tstrerror(code), pstr);





×







1420



        continue;




×







1421



      }










1422












1423



      // extract rows from each stt file one-by-one











UNCOV

1424



      STsdbReader* pReader = pConf->pReader;




×








UNCOV

1425



      int32_t      numOfTables = tSimpleHashGetSize(pReader->status.pTableMap);




×








UNCOV

1426



      uint64_t*    pUidList = pReader->status.uidList.tableUidList;




×








UNCOV

1427



      int32_t      n = 0;




×








UNCOV

1428



      code = getNumOfRowsInSttBlock(pIter->pReader, pIter->pBlockLoadInfo, pStatisBlkArray, pConf->suid, pUidList,




×







1429



                                    numOfTables, &n);











UNCOV

1430



      numOfRows += n;




×








UNCOV

1431



      if (code) {





×







1432



        tsdbError("%s failed to get rows in stt blocks, code:%s", pstr, tstrerror(code));





×







1433



      }










1434



    }










1435



  }










1436













UNCOV

1437



_end:




×








UNCOV

1438



  if (code != TSDB_CODE_SUCCESS) {





×







1439



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));





×







1440



  }











UNCOV

1441



  return numOfRows;




×







1442



}










1443













UNCOV

1444



static bool overlapHelper(const STimeWindow* pLeft, TSKEY minKey, TSKEY maxKey) {




×








UNCOV

1445



  return (pLeft != NULL) && (pLeft->ekey >= minKey) && (pLeft->skey <= maxKey);





×







1446



}










1447













UNCOV

1448



static bool overlapWithTimeWindow(STimeWindow* p1, STimeWindow* pQueryWindow, STableBlockScanInfo* pBlockScanInfo,




×







1449



                                  int32_t order) {











UNCOV

1450



  SIterInfo*   pMemIter = NULL;




×








UNCOV

1451



  SIterInfo*   pIMemIter = NULL;




×








UNCOV

1452



  STbData*     pTbData = NULL;




×








UNCOV

1453



  STimeWindow* pFileWin = NULL;




×







1454













UNCOV

1455



  if (p1 == NULL || pQueryWindow == NULL) {





×







1456



    return false;




×







1457



  }










1458












1459



  // overlap with query window











UNCOV

1460



  if (!(p1->skey >= pQueryWindow->skey && p1->ekey <= pQueryWindow->ekey)) {





×








UNCOV

1461



    return true;




×







1462



  }










1463













UNCOV

1464



  if (pBlockScanInfo == NULL) {





×







1465



    return false;




×







1466



  }










1467













UNCOV

1468



  pMemIter = &pBlockScanInfo->iter;




×








UNCOV

1469



  pIMemIter = &pBlockScanInfo->iiter;




×







1470












1471



  // overlap with mem data











UNCOV

1472



  if (pMemIter->hasVal) {





×








UNCOV

1473



    pTbData = pMemIter->iter->pTbData;




×








UNCOV

1474



    if (overlapHelper(p1, pTbData->minKey, pTbData->maxKey)) {





×








UNCOV

1475



      return true;




×







1476



    }










1477



  }










1478












1479



  // overlap with imem data











UNCOV

1480



  if (pIMemIter->hasVal) {





×








UNCOV

1481



    pTbData = pIMemIter->iter->pTbData;




×








UNCOV

1482



    if (overlapHelper(p1, pTbData->minKey, pTbData->maxKey)) {





×








UNCOV

1483



      return true;




×







1484



    }










1485



  }










1486












1487



  // overlap with data file block











UNCOV

1488



  pFileWin = &pBlockScanInfo->filesetWindow;




×








UNCOV

1489



  if ((taosArrayGetSize(pBlockScanInfo->pBlockIdxList) > 0) && overlapHelper(p1, pFileWin->skey, pFileWin->ekey)) {





×








UNCOV

1490



    return true;




×







1491



  }










1492












1493



  // overlap with deletion skyline











UNCOV

1494



  SBrinRecord record = {.firstKey = p1->skey, .lastKey = p1->ekey};




×








UNCOV

1495



  if (overlapWithDelSkylineWithoutVer(pBlockScanInfo, &record, order)) {





×








UNCOV

1496



    return true;




×







1497



  }










1498













UNCOV

1499



  return false;




×







1500



}










1501













UNCOV

1502



static int32_t sortUidComparFn(const void* p1, const void* p2) {




×








UNCOV

1503



  const SSttKeyRange* px1 = p1;




×








UNCOV

1504



  const SSttKeyRange* px2 = p2;




×







1505













UNCOV

1506



  int32_t ret = tRowKeyCompare(&px1->skey, &px2->skey);




×








UNCOV

1507



  return ret;




×







1508



}










1509













UNCOV

1510



bool isCleanSttBlock(SArray* pKeyRangeList, STimeWindow* pQueryWindow, STableBlockScanInfo* pScanInfo, int32_t order) {




×








UNCOV

1511



  int32_t       num = 0;




×








UNCOV

1512



  SSttKeyRange* pRange = NULL;




×







1513



  STimeWindow   w;










1514













UNCOV

1515



  num = taosArrayGetSize(pKeyRangeList);




×








UNCOV

1516



  if (num == 0) {





×








UNCOV

1517



    return false;




×







1518



  }










1519












1520



  // check if it overlap with del skyline











UNCOV

1521



  taosArraySort(pKeyRangeList, sortUidComparFn);




×







1522













UNCOV

1523



  pRange = taosArrayGet(pKeyRangeList, 0);




×








UNCOV

1524



  if (pRange == NULL) {





×







1525



    return false;




×







1526



  }










1527













UNCOV

1528



  w = (STimeWindow){.skey = pRange->skey.ts, .ekey = pRange->ekey.ts};




×








UNCOV

1529



  if (overlapWithTimeWindow(&w, pQueryWindow, pScanInfo, order)) {





×








UNCOV

1530



    return false;




×







1531



  }










1532













UNCOV

1533



  for (int32_t i = 0; i < num - 1; ++i) {





×








UNCOV

1534



    SSttKeyRange* p1 = taosArrayGet(pKeyRangeList, i);




×








UNCOV

1535



    SSttKeyRange* p2 = taosArrayGet(pKeyRangeList, i + 1);




×








UNCOV

1536



    if (p1 == NULL || p2 == NULL) {





×








UNCOV

1537



      return false;




×







1538



    }










1539













UNCOV

1540



    if (p1->ekey.ts >= p2->skey.ts) {





×








UNCOV

1541



      return false;




×







1542



    }










1543













UNCOV

1544



    w = (STimeWindow){.skey = p2->skey.ts, .ekey = p2->ekey.ts};




×








UNCOV

1545



    bool overlap = overlapWithTimeWindow(&w, pQueryWindow, pScanInfo, order);




×








UNCOV

1546



    if (overlap) {





×







1547



      return false;




×







1548



    }










1549



  }










1550













UNCOV

1551



  return true;




×







1552



}










1553













UNCOV

1554



static bool doCheckDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord,




×







1555



                                    int32_t startIndex) {











UNCOV

1556



  size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline);




×







1557













UNCOV

1558



  for (int32_t i = startIndex; i < num; i += 1) {





×








UNCOV

1559



    TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i);




×








UNCOV

1560



    if (p == NULL) {





×







1561



      return false;




×







1562



    }










1563













UNCOV

1564



    if (p->ts >= pRecord->firstKey.key.ts && p->ts <= pRecord->lastKey.key.ts) {





×








UNCOV

1565



      if (p->version >= pRecord->minVer) {





×








UNCOV

1566



        return true;




×







1567



      }











UNCOV

1568



    } else if (p->ts < pRecord->firstKey.key.ts) {  // p->ts < pBlock->minKey.ts





×








UNCOV

1569



      if (p->version >= pRecord->minVer) {





×








UNCOV

1570



        if (i < num - 1) {





×








UNCOV

1571



          TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1);




×








UNCOV

1572



          if (pnext == NULL) {





×







1573



            return false;




×







1574



          }










1575













UNCOV

1576



          if (pnext->ts >= pRecord->firstKey.key.ts) {





×








UNCOV

1577



            return true;




×







1578



          }










1579



        } else {  // it must be the last point










1580



          if (!(p->version == 0)) {





×







1581



            tsdbError("unexpected version:%" PRId64, p->version);





×







1582



          }










1583



        }










1584



      }










1585



    } else {  // (p->ts > pBlock->maxKey.ts) {











UNCOV

1586



      return false;




×







1587



    }










1588



  }










1589













UNCOV

1590



  return false;




×







1591



}










1592













UNCOV

1593



static bool doCheckDatablockOverlapWithoutVersion(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord,




×







1594



                                                  int32_t startIndex) {











UNCOV

1595



  size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline);




×







1596













UNCOV

1597



  for (int32_t i = startIndex; i < num; i += 1) {





×








UNCOV

1598



    TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i);




×








UNCOV

1599



    if (p == NULL) {





×







1600



      return false;




×







1601



    }










1602













UNCOV

1603



    if (p->ts >= pRecord->firstKey.key.ts && p->ts <= pRecord->lastKey.key.ts) {





×








UNCOV

1604



      return true;




×







1605



    } else if (p->ts < pRecord->firstKey.key.ts) {  // p->ts < pBlock->minKey.ts





×







1606



      if (i < num - 1) {





×







1607



        TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1);




×







1608



        if (pnext == NULL) {





×







1609



          return false;




×







1610



        }










1611












1612



        if (pnext->ts >= pRecord->firstKey.key.ts) {





×







1613



          return true;




×







1614



        }










1615



      }










1616



    } else {  // (p->ts > pBlock->maxKey.ts) {










1617



      return false;




×







1618



    }










1619



  }










1620












1621



  return false;




×







1622



}










1623













UNCOV

1624



bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, int32_t order) {




×








UNCOV

1625



  if (pBlockScanInfo->delSkyline == NULL || (taosArrayGetSize(pBlockScanInfo->delSkyline) == 0)) {





×








UNCOV

1626



    return false;




×







1627



  }










1628












1629



  // ts is not overlap











UNCOV

1630



  TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0);




×








UNCOV

1631



  TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline);




×








UNCOV

1632



  if (pFirst == NULL || pLast == NULL) {





×







1633



    return false;




×







1634



  }










1635













UNCOV

1636



  if (pRecord->firstKey.key.ts > pLast->ts || pRecord->lastKey.key.ts < pFirst->ts) {





×








UNCOV

1637



    return false;




×







1638



  }










1639












1640



  // version is not overlap











UNCOV

1641



  if (ASCENDING_TRAVERSE(order)) {





×








UNCOV

1642



    return doCheckDatablockOverlap(pBlockScanInfo, pRecord, pBlockScanInfo->fileDelIndex);




×







1643



  } else {











UNCOV

1644



    int32_t index = pBlockScanInfo->fileDelIndex;




×








UNCOV

1645



    while (1) {




×








UNCOV

1646



      TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, index);




×








UNCOV

1647



      if (p == NULL) {





×







1648



        return false;




×







1649



      }










1650













UNCOV

1651



      if (p->ts > pRecord->firstKey.key.ts && index > 0) {





×








UNCOV

1652



        index -= 1;




×







1653



      } else {  // find the first point that is smaller than the minKey.ts of dataBlock.











UNCOV

1654



        if (p->ts == pRecord->firstKey.key.ts && p->version < pRecord->maxVer && index > 0) {





×








UNCOV

1655



          index -= 1;




×







1656



        }











UNCOV

1657



        break;




×







1658



      }










1659



    }










1660













UNCOV

1661



    return doCheckDatablockOverlap(pBlockScanInfo, pRecord, index);




×







1662



  }










1663



}










1664













UNCOV

1665



bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, int32_t order) {




×








UNCOV

1666



  if (pBlockScanInfo->delSkyline == NULL || (taosArrayGetSize(pBlockScanInfo->delSkyline) == 0)) {





×








UNCOV

1667



    return false;




×







1668



  }










1669












1670



  // ts is not overlap











UNCOV

1671



  TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0);




×








UNCOV

1672



  TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline);




×








UNCOV

1673



  if (pFirst == NULL || pLast == NULL) {





×







1674



    return false;




×







1675



  }










1676













UNCOV

1677



  if (pRecord->firstKey.key.ts > pLast->ts || pRecord->lastKey.key.ts < pFirst->ts) {





×








UNCOV

1678



    return false;




×







1679



  }










1680












1681



  // version is not overlap











UNCOV

1682



  if (ASCENDING_TRAVERSE(order)) {





×








UNCOV

1683



    return doCheckDatablockOverlapWithoutVersion(pBlockScanInfo, pRecord, pBlockScanInfo->fileDelIndex);




×







1684



  } else {










1685



    int32_t index = pBlockScanInfo->fileDelIndex;




×







1686



    while (1) {




×







1687



      TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, index);




×







1688



      if (p == NULL) {





×







1689



        return false;




×







1690



      }










1691












1692



      if (p->ts > pRecord->firstKey.key.ts && index > 0) {





×







1693



        index -= 1;




×







1694



      } else {  // find the first point that is smaller than the minKey.ts of dataBlock.










1695



        if (p->ts == pRecord->firstKey.key.ts && index > 0) {





×







1696



          index -= 1;




×







1697



        }










1698



        break;




×







1699



      }










1700



    }










1701












1702



    return doCheckDatablockOverlapWithoutVersion(pBlockScanInfo, pRecord, index);




×







1703



  }










1704



}
























    

  • 

    •