taosdata / TDengine / #4917

coverage: 65.42% (+0.02%) from 65.402%
#4917

push

travis-ci

web-flow 
merge: from main to 3.0 branch #34204

31 of 34 new or added lines in 2 files covered. (91.18%)

819 existing lines in 129 files now uncovered.

202679 of 309814 relevant lines covered (65.42%)

116724351.99 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

85.94
/source/dnode/vnode/src/tsdb/tsdbReadUtil.c
1 
/*
2 
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3 
 *
4 
 * This program is free software: you can use, redistribute, and/or modify
5 
 * it under the terms of the GNU Affero General Public License, version 3
6 
 * or later ("AGPL"), as published by the Free Software Foundation.
7 
 *
8 
 * This program is distributed in the hope that it will be useful, but WITHOUT
9 
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 
 * FITNESS FOR A PARTICULAR PURPOSE.
11 
 *
12 
 * You should have received a copy of the GNU Affero General Public License
13 
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
14 
 */
15 









16



#include "tsdbReadUtil.h"










17



#include "tsdb.h"










18



#include "tsdbDataFileRW.h"










19



#include "tsdbFS2.h"










20



#include "tsdbMerge.h"










21



#include "tsdbUtil2.h"










22



#include "tsimplehash.h"










23












24



static bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord,










25



                                            int32_t order);










26












27



static int32_t initBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) {




258,284,944✔





28



  int32_t              code = TSDB_CODE_SUCCESS;




258,284,944✔





29



  int32_t              lino = 0;




258,284,944✔





30



  int32_t              num = 0;




258,284,944✔





31



  int32_t              remainder = 0;




258,284,944✔





32



  STableBlockScanInfo* p = NULL;




258,284,944✔





33



  const void*          px = NULL;




258,400,127✔





34












35



  TSDB_CHECK_CONDITION(pBuf && pBuf->numPerBucket > 0, code, lino, _end, TSDB_CODE_INVALID_PARA);




258,400,127✔





36



  TSDB_CHECK_CONDITION(numOfTables >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA);




257,576,110✔





37












38



  num = numOfTables / pBuf->numPerBucket;




257,576,110✔





39



  remainder = numOfTables % pBuf->numPerBucket;




258,407,731✔





40












41



  if (pBuf->pData == NULL) {




257,918,830✔





42



    pBuf->pData = taosArrayInit(num + 1, POINTER_BYTES);




257,842,512✔





43



    TSDB_CHECK_NULL(pBuf->pData, code, lino, _end, terrno);




258,076,754✔





44



  }










45












46



  for (int32_t i = 0; i < num; ++i) {




258,167,927✔





47



    p = taosMemoryCalloc(pBuf->numPerBucket, sizeof(STableBlockScanInfo));




27,469✔





48



    TSDB_CHECK_NULL(p, code, lino, _end, terrno);




27,469✔





49












50



    px = taosArrayPush(pBuf->pData, &p);




27,469✔





51



    TSDB_CHECK_NULL(px, code, lino, _end, terrno);




27,469✔





52



    p = NULL;




27,469✔





53



  }










54












55



  if (remainder > 0) {




258,140,458✔





56



    p = taosMemoryCalloc(remainder, sizeof(STableBlockScanInfo));




247,043,437✔





57



    TSDB_CHECK_NULL(p, code, lino, _end, terrno);




246,957,192✔





58












59



    px = taosArrayPush(pBuf->pData, &p);




246,957,192✔





60



    TSDB_CHECK_NULL(px, code, lino, _end, terrno);




247,158,860✔





61



    p = NULL;




247,158,860✔





62



  }










63












64



  pBuf->numOfTables = numOfTables;




258,255,881✔





65












66



_end:




258,356,536✔





67



  if (code != TSDB_CODE_SUCCESS) {




258,380,531✔





68



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







69



  }










70



  if (p) {




258,258,007✔





71



    taosMemoryFreeClear(p);




×







72



  }










73



  return code;




258,258,007✔





74



}










75












76



int32_t uidComparFunc(const void* p1, const void* p2) {




1,326,542,798✔





77



  uint64_t pu1 = *(const uint64_t*)p1;




1,326,542,798✔





78



  uint64_t pu2 = *(const uint64_t*)p2;




1,326,570,253✔





79



  if (pu1 == pu2) {




1,326,664,605✔





80



    return 0;




×







81



  } else {










82



    return (pu1 < pu2) ? -1 : 1;




1,326,664,605✔





83



  }










84



}










85












86



int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) {




3,048✔





87



  int32_t              code = TSDB_CODE_SUCCESS;




3,048✔





88



  int32_t              lino = 0;




3,048✔





89



  int32_t              num = 0;




3,048✔





90



  int32_t              remainder = 0;




3,048✔





91



  STableBlockScanInfo* p = NULL;




3,048✔





92



  const void*          px = NULL;




3,048✔





93












94



  TSDB_CHECK_CONDITION(pBuf && pBuf->numPerBucket > 0 && pBuf->numOfTables >= 0, code, lino, _end,




3,048✔





95



                       TSDB_CODE_INVALID_PARA);










96



  TSDB_CHECK_CONDITION(numOfTables >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA);




3,048✔





97












98



  if (numOfTables <= pBuf->numOfTables) {




3,048✔





99



    goto _end;




×







100



  }










101












102



  remainder = pBuf->numOfTables % pBuf->numPerBucket;




3,048✔





103



  if (remainder > 0) {




3,048✔





104



    TSDB_CHECK_CONDITION(taosArrayGetSize(pBuf->pData) > 0, code, lino, _end, TSDB_CODE_INVALID_PARA);




3,048✔





105



    px = taosArrayPop(pBuf->pData);




3,048✔





106



    TSDB_CHECK_NULL(px, code, lino, _end, TSDB_CODE_INVALID_PARA);




3,048✔





107



    p = *(STableBlockScanInfo**)px;




3,048✔





108



    taosMemoryFreeClear(p);




3,048✔





109



    pBuf->numOfTables -= remainder;




3,048✔





110



  }










111












112



  num = (numOfTables - pBuf->numOfTables) / pBuf->numPerBucket;




3,048✔





113



  remainder = (numOfTables - pBuf->numOfTables) % pBuf->numPerBucket;




3,048✔





114












115



  if (pBuf->pData == NULL) {




3,048✔





116



    pBuf->pData = taosArrayInit(num + 1, POINTER_BYTES);




×







117



    TSDB_CHECK_NULL(pBuf->pData, code, lino, _end, terrno);




×







118



  }










119












120



  for (int32_t i = 0; i < num; ++i) {




3,048✔





121



    p = taosMemoryCalloc(pBuf->numPerBucket, sizeof(STableBlockScanInfo));




×







122



    TSDB_CHECK_NULL(p, code, lino, _end, terrno);




×







123












124



    px = taosArrayPush(pBuf->pData, &p);




×







125



    TSDB_CHECK_NULL(px, code, lino, _end, terrno);




×







126



    p = NULL;




×







127



  }










128












129



  if (remainder > 0) {




3,048✔





130



    p = taosMemoryCalloc(remainder, sizeof(STableBlockScanInfo));




3,048✔





131



    TSDB_CHECK_NULL(p, code, lino, _end, terrno);




3,048✔





132












133



    px = taosArrayPush(pBuf->pData, &p);




3,048✔





134



    TSDB_CHECK_NULL(px, code, lino, _end, terrno);




3,048✔





135



    p = NULL;




3,048✔





136



  }










137












138



  pBuf->numOfTables = numOfTables;




3,048✔





139












140



_end:




3,048✔





141



  if (code != TSDB_CODE_SUCCESS) {




3,048✔





142



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







143



  }










144



  if (p) {




3,048✔





145



    taosMemoryFreeClear(p);




×







146



  }










147



  return code;




3,048✔





148



}










149












150



void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf) {




264,069,259✔





151



  if (pBuf == NULL) return;




264,069,259✔





152



  if (pBuf->pData != NULL) {




264,069,259✔





153



    taosArrayDestroyP(pBuf->pData, NULL);




258,578,228✔





154



    pBuf->pData = NULL;




258,407,685✔





155



  }










156



}










157












158



int32_t getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index, STableBlockScanInfo** pInfo) {




621,040,229✔





159



  int32_t               code = TSDB_CODE_SUCCESS;




621,040,229✔





160



  int32_t               lino = 0;




621,040,229✔





161



  int32_t               bucketIndex = 0;




621,040,229✔





162



  STableBlockScanInfo** pBucket = NULL;




621,040,229✔





163












164



  TSDB_CHECK_CONDITION(pBuf && pBuf->numPerBucket > 0, code, lino, _end, TSDB_CODE_INVALID_PARA);




621,040,229✔





165



  TSDB_CHECK_CONDITION(index >= 0 && index < pBuf->numOfTables, code, lino, _end, TSDB_CODE_INVALID_PARA);




620,913,339✔





166



  TSDB_CHECK_NULL(pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




621,086,602✔





167












168



  *pInfo = NULL;




621,086,602✔





169












170



  bucketIndex = index / pBuf->numPerBucket;




621,045,467✔





171



  pBucket = taosArrayGet(pBuf->pData, bucketIndex);




620,991,417✔





172



  TSDB_CHECK_NULL(pBucket, code, lino, _end, terrno);




620,750,979✔





173












174



  *pInfo = (*pBucket) + (index % pBuf->numPerBucket);




620,750,979✔





175












176



_end:




620,932,034✔





177



  if (code != TSDB_CODE_SUCCESS) {




620,932,034✔





178



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







179



  }










180



  return code;




620,689,582✔





181



}










182












183



int32_t getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, STableBlockScanInfo** pInfo, const char* id) {




153,960,078✔





184



  int32_t               code = TSDB_CODE_SUCCESS;




153,960,078✔





185



  int32_t               lino = 0;




153,960,078✔





186



  STableBlockScanInfo** pVal = NULL;




153,960,078✔





187












188



  TSDB_CHECK_NULL(pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




153,960,078✔





189



  TSDB_CHECK_NULL(id, code, lino, _end, TSDB_CODE_INVALID_PARA);




153,960,078✔





190












191



  pVal = (STableBlockScanInfo**)tSimpleHashGet(pTableMap, &uid, sizeof(uid));




153,960,078✔





192



  if (pVal == NULL) {




153,998,608✔





193



    int32_t size = tSimpleHashGetSize(pTableMap);




×







194



    tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id);




×







195



    code = TSDB_CODE_INVALID_PARA;




×







196



    TSDB_CHECK_CODE(code, lino, _end);




×







197



  }










198



  *pInfo = *pVal;




153,998,608✔





199



  TSDB_CHECK_NULL(*pInfo, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);




153,997,909✔





200












201



_end:




153,998,608✔





202



  if (code != TSDB_CODE_SUCCESS) {




153,998,608✔





203



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







204



  }










205



  return code;




153,995,432✔





206



}










207












208



int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc) {




2,147,483,647✔





209



  int32_t code = TSDB_CODE_SUCCESS;




2,147,483,647✔





210



  int32_t lino = 0;




2,147,483,647✔





211












212



  TSDB_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





213












214



  pKey->numOfPKs = numOfPks;




2,147,483,647✔





215



  pKey->ts = ts;




2,147,483,647✔





216












217



  if (numOfPks > 0) {




2,147,483,647✔





218



    pKey->pks[0].type = type;




109,750,765✔





219












220



    if (IS_NUMERIC_TYPE(type)) {




109,748,977✔





221



      if (asc) {




76,519,174✔





222



        switch (type) {




48,512,088✔





223



          case TSDB_DATA_TYPE_BIGINT: {




12,019,469✔





224



            VALUE_SET_TRIVIAL_DATUM(pKey->pks, INT64_MIN);




12,019,469✔





225



            break;




12,020,693✔





226



          }










227



          case TSDB_DATA_TYPE_INT: {




12,619,240✔





228



            int32_t min = INT32_MIN;




12,619,240✔





229



            valueSetDatum(pKey->pks, type, &min, tDataTypes[type].bytes);




12,619,240✔





230



            break;




12,620,198✔





231



          }










232



          case TSDB_DATA_TYPE_SMALLINT: {




×







233



            int16_t min = INT16_MIN;




×







234



            valueSetDatum(pKey->pks, type, &min, tDataTypes[type].bytes);




×







235



            break;




×







236



          }










237



          case TSDB_DATA_TYPE_TINYINT: {




×







238



            int8_t min = INT8_MIN;




×







239



            valueSetDatum(pKey->pks, type, &min, tDataTypes[type].bytes);




×







240



            break;




×







241



          }










242



          case TSDB_DATA_TYPE_UTINYINT:




23,874,409✔





243



          case TSDB_DATA_TYPE_USMALLINT:










244



          case TSDB_DATA_TYPE_UINT:










245



          case TSDB_DATA_TYPE_UBIGINT: {










246



            VALUE_SET_TRIVIAL_DATUM(pKey->pks, 0);




23,874,409✔





247



            break;




23,876,785✔





248



          }











UNCOV

249



          default:




×








UNCOV

250



            code = TSDB_CODE_INVALID_PARA;




×








UNCOV

251



            TSDB_CHECK_CODE(code, lino, _end);




×







252



        }










253



      } else {










254



        switch (type) {




28,007,086✔





255



          case TSDB_DATA_TYPE_BIGINT:




6,923,991✔





256



            VALUE_SET_TRIVIAL_DATUM(pKey->pks, INT64_MAX);




6,923,991✔





257



            break;




6,924,567✔





258



          case TSDB_DATA_TYPE_INT:




7,133,917✔





259



            VALUE_SET_TRIVIAL_DATUM(pKey->pks, INT32_MAX);




7,133,917✔





260



            break;




7,133,917✔





261



          case TSDB_DATA_TYPE_SMALLINT:




×







262



            VALUE_SET_TRIVIAL_DATUM(pKey->pks, INT16_MAX);




×







263



            break;




×







264



          case TSDB_DATA_TYPE_TINYINT:




×







265



            VALUE_SET_TRIVIAL_DATUM(pKey->pks, INT8_MAX);




×







266



            break;




×







267



          case TSDB_DATA_TYPE_UBIGINT:




6,920,355✔





268



            VALUE_SET_TRIVIAL_DATUM(pKey->pks, UINT64_MAX);




6,920,355✔





269



            break;




6,920,355✔





270



          case TSDB_DATA_TYPE_UINT:




7,028,247✔





271



            VALUE_SET_TRIVIAL_DATUM(pKey->pks, UINT32_MAX);




7,028,247✔





272



            break;




7,028,823✔





273



          case TSDB_DATA_TYPE_USMALLINT:




×







274



            VALUE_SET_TRIVIAL_DATUM(pKey->pks, UINT16_MAX);




×







275



            break;




×







276



          case TSDB_DATA_TYPE_UTINYINT:




×







277



            VALUE_SET_TRIVIAL_DATUM(pKey->pks, UINT8_MAX);




×








UNCOV

278



            break;




×







279



          default:




576✔





280



            code = TSDB_CODE_INVALID_PARA;




576✔





281



            TSDB_CHECK_CODE(code, lino, _end);




576✔





282



        }










283



      }










284



    } else {










285



      pKey->pks[0].nData = 0;




33,229,803✔





286












287



      if (pKey->pks[0].pData == NULL) {




33,241,959✔





288



        pKey->pks[0].pData = taosMemoryCalloc(1, len);




33,240,066✔





289



        TSDB_CHECK_NULL(pKey->pks[0].pData, code, lino, _end, terrno);




33,171,003✔





290



      }










291












292



      if (!asc) {




33,184,743✔





293



        pKey->numOfPKs = 2;




13,514,579✔





294



      }










295



    }










296



  }










297












298



_end:




2,147,483,647✔





299



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





300



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







301



  }










302



  return code;




2,147,483,647✔





303



}










304












305



void clearRowKey(SRowKey* pKey) {




2,147,483,647✔





306



  if (pKey == NULL || pKey->numOfPKs == 0 || (!IS_VAR_DATA_TYPE(pKey->pks[0].type))) {




2,147,483,647✔





307



    return;




2,147,483,647✔





308



  }










309



  taosMemoryFreeClear(pKey->pks[0].pData);




20,293,630✔





310



  taosMemoryFreeClear(pKey->pks[1].pData);




38,687,212✔





311



}










312












313



static int32_t initLastProcKey(STableBlockScanInfo* pScanInfo, const STsdbReader* pReader) {




620,996,684✔





314



  int32_t  code = TSDB_CODE_SUCCESS;




620,996,684✔





315



  int32_t  lino = 0;




620,996,684✔





316



  int32_t  numOfPks = 0;




620,996,684✔





317



  int32_t  type = 0;




620,996,684✔





318



  int32_t  bytes = 0;




620,996,684✔





319



  bool     asc = false;




620,996,684✔





320



  SRowKey* pRowKey = NULL;




620,996,684✔





321












322



  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




620,996,684✔





323



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




620,996,684✔





324












325



  numOfPks = pReader->suppInfo.numOfPks;




620,996,684✔





326



  asc = ASCENDING_TRAVERSE(pReader->info.order);




620,808,410✔





327



  type = pReader->suppInfo.pk.type;




620,667,480✔





328



  bytes = pReader->suppInfo.pk.bytes;




620,519,581✔





329












330



  pRowKey = &pScanInfo->lastProcKey;




620,802,280✔





331



  if (asc) {




621,026,774✔





332



    int64_t skey = pReader->info.window.skey;




574,345,259✔





333



    int64_t ts = (skey > INT64_MIN) ? (skey - 1) : skey;




573,894,371✔





334












335



    code = initRowKey(pRowKey, ts, numOfPks, type, bytes, asc);




573,894,371✔





336



    TSDB_CHECK_CODE(code, lino, _end);




573,743,816✔





337












338



    code = initRowKey(&pScanInfo->sttKeyInfo.nextProcKey, skey, numOfPks, type, bytes, asc);




573,743,816✔





339



    TSDB_CHECK_CODE(code, lino, _end);




574,333,560✔





340



  } else {










341



    int64_t ekey = pReader->info.window.ekey;




46,681,515✔





342



    int64_t ts = (ekey < INT64_MAX) ? (ekey + 1) : ekey;




46,677,320✔





343












344



    code = initRowKey(pRowKey, ts, numOfPks, type, bytes, asc);




46,677,320✔





345



    TSDB_CHECK_CODE(code, lino, _end);




46,667,774✔





346












347



    code = initRowKey(&pScanInfo->sttKeyInfo.nextProcKey, ekey, numOfPks, type, bytes, asc);




46,667,774✔





348



    TSDB_CHECK_CODE(code, lino, _end);




46,543,895✔





349



  }










350












351



  code = initRowKey(&pScanInfo->sttRange.skey, INT64_MAX, numOfPks, type, bytes, asc);




620,877,455✔





352



  TSDB_CHECK_CODE(code, lino, _end);




621,128,253✔





353












354



  code = initRowKey(&pScanInfo->sttRange.ekey, INT64_MIN, numOfPks, type, bytes, asc);




621,128,253✔





355



  TSDB_CHECK_CODE(code, lino, _end);




620,982,879✔





356












357



_end:




620,982,879✔





358



  if (code != TSDB_CODE_SUCCESS) {




620,982,879✔





359



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







360



  }










361



  return code;




620,971,369✔





362



}










363












364



int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSHashObj* pTableMap,




621,072,822✔





365



                               STsdbReader* pReader) {










366



  int32_t code = TSDB_CODE_SUCCESS;




621,072,822✔





367



  int32_t lino = 0;




621,072,822✔





368












369



  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




621,072,822✔





370



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




621,072,822✔





371












372



  pScanInfo->uid = uid;




621,072,822✔





373



  INIT_KEYRANGE(&pScanInfo->sttRange);




620,949,231✔





374



  INIT_TIMEWINDOW(&pScanInfo->filesetWindow);




620,975,629✔





375












376



  pScanInfo->cleanSttBlocks = false;




620,570,543✔





377



  pScanInfo->sttBlockReturned = false;




621,042,448✔





378












379



  code = initLastProcKey(pScanInfo, pReader);




620,881,679✔





380



  TSDB_CHECK_CODE(code, lino, _end);




620,944,340✔





381












382



  pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT;




620,944,340✔





383



  code = tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES);




620,914,919✔





384



  TSDB_CHECK_CODE(code, lino, _end);




620,793,827✔





385












386



  tsdbTrace("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pReader, pScanInfo->uid,




620,793,827✔





387



            pScanInfo->lastProcKey.ts, pReader->idStr);










388












389



_end:




612,324,834✔





390



  if (code != TSDB_CODE_SUCCESS) {




620,995,925✔





391



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







392



  }










393



  return code;




620,655,331✔





394



}










395












396



// NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model










397



int32_t createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList,




258,293,940✔





398



                                STableUidList* pUidList, int32_t numOfTables, SSHashObj** pHashObj) {










399



  int32_t    code = TSDB_CODE_SUCCESS;




258,293,940✔





400



  int32_t    lino = 0;




258,293,940✔





401



  SSHashObj* pTableMap = NULL;




258,293,940✔





402



  int64_t    st = 0;




258,293,940✔





403












404



  TSDB_CHECK_NULL(pUidList, code, lino, _end, TSDB_CODE_INVALID_PARA);




258,293,940✔





405



  TSDB_CHECK_NULL(pHashObj, code, lino, _end, TSDB_CODE_INVALID_PARA);




258,293,940✔





406












407



  *pHashObj = NULL;




258,293,940✔





408












409



  // allocate buffer in order to load data blocks from file










410



  // todo use simple hash instead, optimize the memory consumption










411



  pTableMap = tSimpleHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));




258,217,575✔





412



  TSDB_CHECK_NULL(pTableMap, code, lino, _end, terrno);




258,046,688✔





413












414



  st = taosGetTimestampUs();




258,352,316✔





415



  code = initBlockScanInfoBuf(pBuf, numOfTables);




258,352,316✔





416



  if (code != TSDB_CODE_SUCCESS) {




258,236,217✔





417



    tSimpleHashCleanup(pTableMap);




×







418



    TSDB_CHECK_CODE(code, lino, _end);




×







419



  }










420












421



  pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t));




258,236,217✔





422



  if (pUidList->tableUidList == NULL) {




258,277,201✔





423



    tSimpleHashCleanup(pTableMap);




×







424



    TSDB_CHECK_NULL(pUidList->tableUidList, code, lino, _end, terrno);




×







425



  }










426












427



  pUidList->currentIndex = 0;




258,182,672✔





428












429



  for (int32_t j = 0; j < numOfTables; ++j) {




873,341,716✔





430



    pUidList->tableUidList[j] = idList[j].uid;




615,183,392✔





431












432



    STableBlockScanInfo* pScanInfo = NULL;




615,152,112✔





433



    code = getPosInBlockInfoBuf(pBuf, j, &pScanInfo);




615,164,237✔





434



    if (code != TSDB_CODE_SUCCESS) {




614,580,629✔





435



      lino = __LINE__;




×







436



      break;




×







437



    }










438












439



    code = initTableBlockScanInfo(pScanInfo, idList[j].uid, pTableMap, pTsdbReader);




614,580,629✔





440



    if (code != TSDB_CODE_SUCCESS) {




614,855,745✔





441



      lino = __LINE__;




×







442



      break;




×







443



    }










444



  }










445












446



  taosSort(pUidList->tableUidList, numOfTables, sizeof(uint64_t), uidComparFunc);




258,102,736✔





447












448



  pTsdbReader->cost.createScanInfoList = (taosGetTimestampUs() - st) / 1000.0;




258,541,810✔





449



  tsdbDebug("%p create %d tables scan-info, size:%.2f Kb, elapsed time:%.2f ms, %s", pTsdbReader, numOfTables,




258,384,506✔





450



            (sizeof(STableBlockScanInfo) * numOfTables) / 1024.0, pTsdbReader->cost.createScanInfoList,










451



            pTsdbReader->idStr);










452












453



  *pHashObj = pTableMap;




258,385,240✔





454












455



_end:




258,422,559✔





456



  if (code != TSDB_CODE_SUCCESS) {




258,422,559✔





457



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







458



  }










459



  return code;




258,375,978✔





460



}










461












462



void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) {




8,939,969✔





463



  STableBlockScanInfo** p = NULL;




8,939,969✔





464



  int32_t               iter = 0;




8,939,969✔





465












466



  while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) {




18,011,208✔





467



    STableBlockScanInfo* pInfo = *p;




9,071,239✔





468



    if (pInfo == NULL) {




9,071,239✔





469



      continue;




×







470



    }










471












472



    pInfo->iterInit = false;




9,071,239✔





473



    pInfo->iter.hasVal = false;




9,071,239✔





474



    pInfo->iiter.hasVal = false;




9,070,719✔





475












476



    if (pInfo->iter.iter != NULL) {




9,070,739✔





477



      pInfo->iter.iter = tsdbTbDataIterDestroy(pInfo->iter.iter);




2,697,387✔





478



    }










479












480



    if (pInfo->iiter.iter != NULL) {




9,070,739✔





481



      pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter);




1,093✔





482



    }










483












484



    taosArrayDestroy(pInfo->delSkyline);




9,070,739✔





485



    pInfo->delSkyline = NULL;




9,070,719✔





486



    pInfo->lastProcKey.ts = ts;




9,071,239✔





487



    // todo check the nextProcKey info










488



    pInfo->sttKeyInfo.nextProcKey.ts = ts + step;




9,070,719✔





489



  }










490



}




8,939,469✔





491












492



// keep the lastProcKey/sttRange/nextProcKey info










493



void clearBlockScanInfoLoadInfo(STableBlockScanInfo* p) {




621,289,284✔





494



  if (p == NULL) {




621,289,284✔





495



    return;




×







496



  }










497












498



  p->iterInit = false;




621,289,284✔





499



  p->iter.hasVal = false;




621,364,827✔





500



  p->iiter.hasVal = false;




621,263,623✔





501



  p->sttKeyInfo.status = STT_FILE_READER_UNINIT;




620,898,597✔





502












503



  if (p->iter.iter != NULL) {




621,127,344✔





504



    p->iter.iter = tsdbTbDataIterDestroy(p->iter.iter);




384,029,197✔





505



  }










506












507



  if (p->iiter.iter != NULL) {




620,886,207✔





508



    p->iiter.iter = tsdbTbDataIterDestroy(p->iiter.iter);




3,433,249✔





509



  }










510












511



  taosArrayDestroy(p->delSkyline);




621,192,698✔





512



  p->delSkyline = NULL;




621,227,484✔





513



  taosArrayDestroy(p->pBlockList);




621,307,300✔





514



  p->pBlockList = NULL;




621,083,950✔





515



  taosArrayDestroy(p->pBlockIdxList);




621,108,835✔





516



  p->pBlockIdxList = NULL;




621,106,207✔





517



  taosArrayDestroy(p->pMemDelData);




621,175,911✔





518



  p->pMemDelData = NULL;




621,078,094✔





519



  taosArrayDestroy(p->pFileDelData);




621,086,808✔





520



  p->pFileDelData = NULL;




621,100,869✔





521



}










522












523



void clearBlockScanInfo(STableBlockScanInfo* p) {




621,296,997✔





524



  if (p == NULL) {




621,296,997✔





525



    return;




×







526



  }










527












528



  clearBlockScanInfoLoadInfo(p);




621,296,997✔





529












530



  clearRowKey(&p->lastProcKey);




621,121,014✔





531



  clearRowKey(&p->sttRange.skey);




621,199,929✔





532



  clearRowKey(&p->sttRange.ekey);




621,166,120✔





533



  clearRowKey(&p->sttKeyInfo.nextProcKey);




621,155,165✔





534



}










535












536



void destroyAllBlockScanInfo(SSHashObj** pTableMap) {




258,514,199✔





537



  STableBlockScanInfo** p = NULL;




258,514,199✔





538



  int32_t               iter = 0;




258,514,199✔





539












540



  if (pTableMap == NULL || *pTableMap == NULL) {




258,556,065✔





541



    return;




×







542



  }










543












544



  while ((p = tSimpleHashIterate(*pTableMap, p, &iter)) != NULL) {




873,940,907✔





545



    clearBlockScanInfo(*p);




615,270,217✔





546



  }










547












548



  tSimpleHashCleanup(*pTableMap);




258,536,935✔





549



  *pTableMap = NULL;




258,435,402✔





550



}










551












552



static void doCleanupInfoForNextFileset(STableBlockScanInfo* pScanInfo) {




581,182,080✔





553



  if (pScanInfo == NULL) {




581,182,080✔





554



    return;




×







555



  }










556



  // reset the index in last block when handing a new file










557



  taosArrayClear(pScanInfo->pBlockList);




581,182,080✔





558



  taosArrayClear(pScanInfo->pBlockIdxList);




581,216,192✔





559



  taosArrayClear(pScanInfo->pFileDelData);  // del data from each file set




581,234,837✔





560



  pScanInfo->cleanSttBlocks = false;




581,297,310✔





561



  pScanInfo->numOfRowsInStt = 0;




581,243,933✔





562



  pScanInfo->sttBlockReturned = false;




581,254,801✔





563



  INIT_KEYRANGE(&pScanInfo->sttRange);




581,297,044✔





564



  INIT_TIMEWINDOW(&pScanInfo->filesetWindow);




581,260,495✔





565



  pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT;




581,209,095✔





566



}










567












568



void cleanupInfoForNextFileset(SSHashObj* pTableMap) {




342,302,240✔





569



  STableBlockScanInfo** p = NULL;




342,302,240✔





570



  int32_t               iter = 0;




342,302,240✔





571












572



  while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) {




923,589,338✔





573



    doCleanupInfoForNextFileset(*p);




581,194,170✔





574



  }










575



}




342,510,371✔





576












577



// brin records iterator










578



void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList) {




342,480,267✔





579



  if (pIter == NULL) {




342,480,267✔





580



    return;




×







581



  }










582












583



  (void)memset(&pIter->block, 0, sizeof(SBrinBlock));




342,480,267✔





584



  (void)memset(&pIter->record, 0, sizeof(SBrinRecord));




342,522,225✔





585



  pIter->blockIndex = -1;




342,382,462✔





586



  pIter->recordIndex = -1;




342,498,312✔





587












588



  pIter->pReader = pReader;




342,491,709✔





589



  pIter->pBrinBlockList = pList;




342,458,834✔





590



}










591












592



int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord) {




397,023,771✔





593



  int32_t code = TSDB_CODE_SUCCESS;




397,023,771✔





594



  int32_t lino = 0;




397,023,771✔





595












596



  TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA);




397,023,771✔





597



  TSDB_CHECK_NULL(pRecord, code, lino, _end, TSDB_CODE_INVALID_PARA);




397,023,771✔





598












599



  *pRecord = NULL;




397,023,771✔





600












601



  if (pIter->blockIndex == -1 || (pIter->recordIndex + 1) >= pIter->block.numOfRecords) {




397,096,355✔





602



    pIter->blockIndex += 1;




356,144,664✔





603



    if (pIter->blockIndex >= taosArrayGetSize(pIter->pBrinBlockList)) {




356,062,719✔





604



      goto _end;




340,476,292✔





605



    }










606












607



    pIter->pCurrentBlk = taosArrayGet(pIter->pBrinBlockList, pIter->blockIndex);




15,634,072✔





608



    TSDB_CHECK_NULL(pIter->pCurrentBlk, code, lino, _end, terrno);




15,634,339✔





609












610



    tBrinBlockClear(&pIter->block);




15,633,031✔





611



    TSDB_CHECK_NULL(pIter->pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




15,630,415✔





612



    code = tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block);




15,634,166✔





613



    if (code != TSDB_CODE_SUCCESS) {




15,635,910✔





614



      tsdbError("failed to read brinBlock from file, code:%s", tstrerror(code));




×







615



      TSDB_CHECK_CODE(code, lino, _end);




×







616



    }










617












618



    pIter->recordIndex = -1;




15,635,910✔





619



  }










620












621



  pIter->recordIndex += 1;




56,604,095✔





622



  code = tBrinBlockGet(&pIter->block, pIter->recordIndex, &pIter->record);




56,602,266✔





623



  *pRecord = &pIter->record;




56,585,379✔





624



  TSDB_CHECK_CODE(code, lino, _end);




56,585,905✔





625












626



_end:




56,585,905✔





627



  if (code != TSDB_CODE_SUCCESS) {




397,062,197✔





628



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







629



  }










630



  return code;




396,818,558✔





631



}










632












633



void clearBrinBlockIter(SBrinRecordIter* pIter) {




342,217,705✔





634



  if (pIter != NULL) {




342,217,705✔





635



    tBrinBlockDestroy(&pIter->block);




342,308,228✔





636



  }










637



}




342,357,036✔





638












639



// initialize the file block access order










640



//  sort the file blocks according to the offset of each data block in the files










641



static void cleanupBlockOrderSupporter(SBlockOrderSupporter* pSup) {




15,510,980✔





642



  if (pSup == NULL) {




15,510,980✔





643



    return;




×







644



  }










645












646



  taosMemoryFreeClear(pSup->numOfBlocksPerTable);




15,510,980✔





647



  taosMemoryFreeClear(pSup->indexPerTable);




15,512,724✔





648












649



  if (pSup->pDataBlockInfo != NULL) {




15,510,450✔





650



    for (int32_t i = 0; i < pSup->numOfTables; ++i) {




51,036,397✔





651



      SBlockOrderWrapper* pBlockInfo = pSup->pDataBlockInfo[i];




35,524,109✔





652



      taosMemoryFreeClear(pBlockInfo);




35,524,545✔





653



    }










654












655



    taosMemoryFreeClear(pSup->pDataBlockInfo);




15,512,288✔





656



  }










657



}










658












659



static int32_t initBlockOrderSupporter(SBlockOrderSupporter* pSup, int32_t numOfTables) {




15,510,980✔





660



  int32_t code = TSDB_CODE_SUCCESS;




15,510,980✔





661



  int32_t lino = 0;




15,510,980✔





662












663



  TSDB_CHECK_NULL(pSup, code, lino, _end, TSDB_CODE_INVALID_PARA);




15,510,980✔





664



  TSDB_CHECK_CONDITION(numOfTables >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA);




15,510,980✔





665












666



  pSup->pDataBlockInfo = taosMemoryCalloc(numOfTables, POINTER_BYTES);




15,510,980✔





667



  TSDB_CHECK_NULL(pSup->pDataBlockInfo, code, lino, _end, terrno);




15,511,416✔





668



  pSup->indexPerTable = taosMemoryCalloc(numOfTables, sizeof(int32_t));




15,510,544✔





669



  TSDB_CHECK_NULL(pSup->indexPerTable, code, lino, _end, terrno);




15,514,032✔





670



  pSup->numOfBlocksPerTable = taosMemoryCalloc(numOfTables, sizeof(int32_t));




15,513,066✔





671



  TSDB_CHECK_NULL(pSup->numOfBlocksPerTable, code, lino, _end, terrno);




15,511,852✔





672



  pSup->numOfTables = 0;




15,512,724✔





673












674



_end:




15,511,852✔





675



  if (code != TSDB_CODE_SUCCESS) {




15,511,852✔





676



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







677



  }










678



  return code;




15,511,852✔





679



}










680












681



static int32_t fileDataBlockOrderCompar(const void* pLeft, const void* pRight, void* param) {




100,284,514✔





682



  int32_t                     leftIndex = 0;




100,284,514✔





683



  int32_t                     rightIndex = 0;




100,284,514✔





684



  int32_t                     leftTableBlockIndex = 0;




100,284,514✔





685



  int32_t                     rightTableBlockIndex = 0;




100,284,514✔





686



  const SBlockOrderSupporter* pSupporter = NULL;




100,284,514✔





687



  const SBlockOrderWrapper*   pLeftBlock = NULL;




100,284,514✔





688



  const SBlockOrderWrapper*   pRightBlock = NULL;




100,284,514✔





689












690



  leftIndex = *(const int32_t*)pLeft;




100,284,514✔





691



  rightIndex = *(const int32_t*)pRight;




100,285,386✔





692



  pSupporter = (const SBlockOrderSupporter*)param;




100,285,386✔





693












694



  leftTableBlockIndex = pSupporter->indexPerTable[leftIndex];




100,285,386✔





695



  rightTableBlockIndex = pSupporter->indexPerTable[rightIndex];




100,284,833✔





696












697



  if (leftTableBlockIndex >= pSupporter->numOfBlocksPerTable[leftIndex]) {




100,282,695✔





698



    /* left block is empty */










699



    return 1;




42,797,621✔





700



  } else if (rightTableBlockIndex >= pSupporter->numOfBlocksPerTable[rightIndex]) {




57,492,828✔





701



    /* right block is empty */










702



    return -1;




20,012,257✔





703



  }










704












705



  pLeftBlock = &pSupporter->pDataBlockInfo[leftIndex][leftTableBlockIndex];




37,480,890✔





706



  pRightBlock = &pSupporter->pDataBlockInfo[rightIndex][rightTableBlockIndex];




37,480,773✔





707












708



  return pLeftBlock->offset > pRightBlock->offset ? 1 : -1;




37,480,018✔





709



}










710












711



int32_t recordToBlockInfo(SFileDataBlockInfo* pBlockInfo, SBrinRecord* record) {




52,120,793✔





712



  int32_t        code = TSDB_CODE_SUCCESS;




52,120,793✔





713



  int32_t        lino = 0;




52,120,793✔





714



  const SRowKey* pFirstKey = NULL;




52,120,793✔





715



  const SRowKey* pLastKey = NULL;




52,120,793✔





716












717



  TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




52,120,793✔





718



  TSDB_CHECK_NULL(record, code, lino, _end, TSDB_CODE_INVALID_PARA);




52,120,793✔





719












720



  pBlockInfo->uid = record->uid;




52,120,793✔





721



  pBlockInfo->firstKey = record->firstKey.key.ts;




52,126,208✔





722



  pBlockInfo->lastKey = record->lastKey.key.ts;




52,122,218✔





723



  pBlockInfo->minVer = record->minVer;




52,122,601✔





724



  pBlockInfo->maxVer = record->maxVer;




52,122,282✔





725



  pBlockInfo->blockOffset = record->blockOffset;




52,124,435✔





726



  pBlockInfo->smaOffset = record->smaOffset;




52,122,044✔





727



  pBlockInfo->blockSize = record->blockSize;




52,120,623✔





728



  pBlockInfo->blockKeySize = record->blockKeySize;




52,118,786✔





729



  pBlockInfo->smaSize = record->smaSize;




52,120,296✔





730



  pBlockInfo->numRow = record->numRow;




52,118,842✔





731



  pBlockInfo->count = record->count;




52,122,627✔





732












733



  pFirstKey = &record->firstKey.key;




52,122,517✔





734



  pLastKey = &record->lastKey.key;




52,119,573✔





735



  TSDB_CHECK_CONDITION((pFirstKey->numOfPKs == pLastKey->numOfPKs), code, lino, _end, TSDB_CODE_INVALID_PARA);




52,122,101✔





736



  if (pFirstKey->numOfPKs > 0) {




52,117,946✔





737



    if (IS_NUMERIC_TYPE(pFirstKey->pks[0].type)) {




1,330✔





738



      pBlockInfo->firstPk.val = VALUE_GET_TRIVIAL_DATUM(pFirstKey->pks);




×







739



      pBlockInfo->lastPk.val = VALUE_GET_TRIVIAL_DATUM(pLastKey->pks);




×







740



    } else {










741



      int32_t keyLen = pFirstKey->pks[0].nData;




1,330✔





742



      char*   p = taosMemoryMalloc(keyLen + VARSTR_HEADER_SIZE);




1,330✔





743



      TSDB_CHECK_NULL(p, code, lino, _end, terrno);




1,330✔





744



      memcpy(varDataVal(p), pFirstKey->pks[0].pData, keyLen);




1,330✔





745



      varDataSetLen(p, keyLen);




1,330✔





746



      pBlockInfo->firstPk.pData = (uint8_t*)p;




1,330✔





747












748



      keyLen = pLastKey->pks[0].nData;




1,330✔





749



      p = taosMemoryMalloc(keyLen + VARSTR_HEADER_SIZE);




1,330✔





750



      TSDB_CHECK_NULL(p, code, lino, _end, terrno);




1,330✔





751



      memcpy(varDataVal(p), pLastKey->pks[0].pData, keyLen);




1,330✔





752



      varDataSetLen(p, keyLen);




1,330✔





753



      pBlockInfo->lastPk.pData = (uint8_t*)p;




1,330✔





754



    }










755



  }










756












757



_end:




52,117,719✔





758



  if (code != TSDB_CODE_SUCCESS) {




52,119,049✔





759



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







760



  }










761



  return code;




52,116,986✔





762



}










763












764



static void freePkItem(void* pItem) {




1,330✔





765



  SFileDataBlockInfo* p = pItem;




1,330✔





766



  if (p != NULL) {




1,330✔





767



    taosMemoryFreeClear(p->firstPk.pData);




1,330✔





768



    taosMemoryFreeClear(p->lastPk.pData);




1,330✔





769



  }










770



}




1,330✔





771












772



void clearDataBlockIterator(SDataBlockIter* pIter, bool needFree) {




363,485,948✔





773



  if (pIter == NULL) {




363,485,948✔





774



    return;




×







775



  }










776












777



  pIter->index = -1;




363,485,948✔





778



  pIter->numOfBlocks = 0;




363,718,238✔





779












780



  if (needFree) {




363,587,340✔





781



    taosArrayClearEx(pIter->blockList, freePkItem);




44,851,368✔





782



  } else {










783



    taosArrayClear(pIter->blockList);




318,735,972✔





784



  }










785



}










786












787



void cleanupDataBlockIterator(SDataBlockIter* pIter, bool needFree) {




264,085,120✔





788



  if (pIter == NULL) {




264,085,120✔





789



    return;




×







790



  }










791












792



  pIter->index = -1;




264,085,120✔





793



  pIter->numOfBlocks = 0;




264,178,664✔





794



  if (needFree) {




264,073,755✔





795



    taosArrayDestroyEx(pIter->blockList, freePkItem);




4,143,377✔





796



  } else {










797



    taosArrayDestroy(pIter->blockList);




259,930,378✔





798



  }










799



  pIter->blockList = NULL;




263,916,518✔





800



}










801












802



int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList) {




15,510,544✔





803



  int32_t                   code = TSDB_CODE_SUCCESS;




15,510,544✔





804



  int32_t                   lino = 0;




15,510,544✔





805



  bool                      asc = false;




15,510,544✔





806



  int32_t                   numOfTables = 0;




15,510,544✔





807



  int64_t                   st = 0;




15,510,544✔





808



  int64_t                   et = 0;




15,510,544✔





809



  int32_t                   cnt = 0;




15,510,544✔





810



  SBlockOrderSupporter      sup = {0};




15,510,544✔





811



  SMultiwayMergeTreeInfo*   pTree = NULL;




15,513,938✔





812



  STableBlockScanInfo*      pTableScanInfo = NULL;




15,513,160✔





813



  const SFileDataBlockInfo* pBlockInfo = NULL;




15,513,160✔





814



  const void*               px = NULL;




15,513,160✔





815












816



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




15,513,160✔





817



  TSDB_CHECK_NULL(pBlockIter, code, lino, _end, TSDB_CODE_INVALID_PARA);




15,513,160✔





818



  TSDB_CHECK_CONDITION(numOfBlocks >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA);




15,513,160✔





819












820



  asc = ASCENDING_TRAVERSE(pReader->info.order);




15,513,160✔





821



  clearDataBlockIterator(pBlockIter, shouldFreePkBuf(&pReader->suppInfo));




15,514,374✔





822



  pBlockIter->numOfBlocks = numOfBlocks;




15,513,502✔





823












824



  // access data blocks according to the offset of each block in asc/desc order.










825



  numOfTables = taosArrayGetSize(pTableList);




15,513,502✔





826












827



  st = taosGetTimestampUs();




15,514,032✔





828



  code = initBlockOrderSupporter(&sup, numOfTables);




15,514,032✔





829



  TSDB_CHECK_CODE(code, lino, _end);




15,509,236✔





830












831



  for (int32_t i = 0; i < numOfTables; ++i) {




51,032,473✔





832



    pTableScanInfo = taosArrayGetP(pTableList, i);




35,522,365✔





833












834



    size_t              num = taosArrayGetSize(pTableScanInfo->pBlockList);




35,526,289✔





835



    SBlockOrderWrapper* buf = taosMemoryMalloc(sizeof(SBlockOrderWrapper) * num);




35,525,853✔





836



    TSDB_CHECK_NULL(buf, code, lino, _end, terrno);




35,523,237✔





837



    sup.numOfBlocksPerTable[sup.numOfTables] = num;




35,523,237✔





838



    sup.pDataBlockInfo[sup.numOfTables] = buf;




35,524,451✔





839



    sup.numOfTables++;




35,523,579✔





840












841



    for (int32_t k = 0; k < num; ++k) {




87,653,583✔





842



      pBlockInfo = taosArrayGet(pTableScanInfo->pBlockList, k);




52,130,346✔





843



      TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




52,130,688✔





844












845



      sup.pDataBlockInfo[i][k] =




104,260,598✔





846



          (SBlockOrderWrapper){.uid = pTableScanInfo->uid, .offset = pBlockInfo->blockOffset, .pInfo = pTableScanInfo};




52,129,380✔





847



      cnt++;




52,130,004✔





848



    }










849



  }










850












851



  TSDB_CHECK_CONDITION(!(numOfBlocks != cnt && sup.numOfTables != numOfTables), code, lino, _end,




15,510,108✔





852



                       TSDB_CODE_INVALID_PARA);










853












854



  // since there is only one table qualified, blocks are not sorted










855



  if (sup.numOfTables == 1) {




15,510,108✔





856



    pTableScanInfo = taosArrayGetP(pTableList, 0);




8,523,850✔





857



    for (int32_t i = 0; i < numOfBlocks; ++i) {




19,341,722✔





858



      STableDataBlockIdx tableDataBlockIdx = {.globalIndex = i};




10,817,000✔





859



      px = taosArrayPush(pTableScanInfo->pBlockIdxList, &tableDataBlockIdx);




10,815,598✔





860



      TSDB_CHECK_NULL(px, code, lino, _end, terrno);




10,816,564✔





861



    }










862












863



    px = taosArrayAddAll(pBlockIter->blockList, pTableScanInfo->pBlockList);




8,524,722✔





864



    TSDB_CHECK_NULL(px, code, lino, _end, terrno);




8,526,466✔





865












866



    taosArrayDestroy(pTableScanInfo->pBlockList);




8,526,466✔





867



    pTableScanInfo->pBlockList = NULL;




8,527,338✔





868












869



    et = taosGetTimestampUs();




8,526,902✔





870



    tsdbDebug("%p create blocks info struct completed for one table, %d blocks not sorted, elapsed time:%.2f ms %s",




8,526,902✔





871



              pReader, numOfBlocks, (et - st) / 1000.0, pReader->idStr);










872












873



    pBlockIter->index = asc ? 0 : (numOfBlocks - 1);




8,524,722✔





874



    goto _end;




8,525,158✔





875



  }










876












877



  tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables %s", pReader, cnt, sup.numOfTables,




6,986,258✔





878



            pReader->idStr);










879












880



  code = tMergeTreeCreate(&pTree, sup.numOfTables, &sup, fileDataBlockOrderCompar);




6,986,258✔





881



  TSDB_CHECK_CODE(code, lino, _end);




6,987,566✔





882












883



  for (int32_t i = 0; i < cnt; ++i) {




48,294,796✔





884



    int32_t pos = tMergeTreeGetChosenIndex(pTree);




41,306,794✔





885



    int32_t index = sup.indexPerTable[pos]++;




41,306,794✔





886



    pTableScanInfo = sup.pDataBlockInfo[pos][index].pInfo;




41,306,911✔





887












888



    pBlockInfo = taosArrayGet(pTableScanInfo->pBlockList, index);




41,307,347✔





889



    TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, terrno);




41,306,879✔





890












891



    px = taosArrayPush(pBlockIter->blockList, pBlockInfo);




41,306,879✔





892



    TSDB_CHECK_NULL(px, code, lino, _end, terrno);




41,305,858✔





893












894



    STableDataBlockIdx tableDataBlockIdx = {.globalIndex = i};




41,305,858✔





895



    px = taosArrayPush(pTableScanInfo->pBlockIdxList, &tableDataBlockIdx);




41,305,858✔





896



    TSDB_CHECK_NULL(px, code, lino, _end, terrno);




41,303,316✔





897












898



    // set data block index overflow, in order to disable the offset comparator










899



    if (sup.indexPerTable[pos] >= sup.numOfBlocksPerTable[pos]) {




41,303,316✔





900



      sup.indexPerTable[pos] = sup.numOfBlocksPerTable[pos] + 1;




26,998,951✔





901



    }










902












903



    code = tMergeTreeAdjust(pTree, tMergeTreeGetAdjustIndex(pTree));




41,303,082✔





904



    TSDB_CHECK_CODE(code, lino, _end);




41,307,230✔





905



  }










906












907



  for (int32_t i = 0; i < numOfTables; ++i) {




33,987,389✔





908



    pTableScanInfo = taosArrayGetP(pTableList, i);




26,999,823✔





909



    taosArrayDestroy(pTableScanInfo->pBlockList);




26,999,823✔





910



    pTableScanInfo->pBlockList = NULL;




26,999,823✔





911



  }










912












913



  et = taosGetTimestampUs();




6,987,566✔





914



  tsdbDebug("%p %d data blocks access order completed, elapsed time:%.2f ms %s", pReader, numOfBlocks,




6,987,566✔





915



            (et - st) / 1000.0, pReader->idStr);










916












917



  pBlockIter->index = asc ? 0 : (numOfBlocks - 1);




6,987,566✔





918












919



_end:




15,512,724✔





920



  if (code != TSDB_CODE_SUCCESS) {




15,510,108✔





921



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







922



  }










923



  cleanupBlockOrderSupporter(&sup);




15,510,108✔





924



  if (pTree != NULL) {




15,510,108✔





925



    tMergeTreeDestroy(&pTree);




6,987,566✔





926



  }










927



  return code;




15,510,108✔





928



}










929












930



bool blockIteratorNext(SDataBlockIter* pBlockIter) {




51,753,490✔





931



  bool asc = false;




51,753,490✔





932












933



  if (pBlockIter == NULL) {




51,753,490✔





934



    return false;




×







935



  }










936












937



  asc = ASCENDING_TRAVERSE(pBlockIter->order);




51,753,490✔





938



  if ((pBlockIter->index >= pBlockIter->numOfBlocks - 1 && asc) || (pBlockIter->index <= 0 && (!asc))) {




51,759,594✔





939



    return false;




15,411,343✔





940



  }










941



  pBlockIter->index += asc ? 1 : -1;




36,348,687✔





942



  return true;




36,345,752✔





943



}










944












945



typedef enum {










946



  BLK_CHECK_CONTINUE = 0x1,










947



  BLK_CHECK_QUIT = 0x2,










948



} ETombBlkCheckEnum;










949












950



static int32_t loadNextStatisticsBlock(SSttFileReader* pSttFileReader, STbStatisBlock* pStatisBlock,










951



                                       const TStatisBlkArray* pStatisBlkArray, int32_t numOfRows, int32_t* i,










952



                                       int32_t* j);










953



static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_t numOfTables, int32_t* j,




7,387,338✔





954



                                ETombBlkCheckEnum* pRet) {










955



  int32_t              code = TSDB_CODE_SUCCESS;




7,387,338✔





956



  int32_t              lino = 0;




7,387,338✔





957



  STombRecord          record = {0};




7,387,338✔





958



  uint64_t             uid = 0;




7,387,338✔





959



  STableBlockScanInfo* pScanInfo = NULL;




7,387,338✔





960












961



  TSDB_CHECK_NULL(pBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);




7,387,338✔





962



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




7,387,338✔





963



  TSDB_CHECK_NULL(pRet, code, lino, _end, TSDB_CODE_INVALID_PARA);




7,387,338✔





964












965



  *pRet = BLK_CHECK_QUIT;




7,387,338✔





966



  uid = pReader->status.uidList.tableUidList[*j];




7,387,338✔





967



  code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr);




7,387,338✔





968



  TSDB_CHECK_CODE(code, lino, _end);




7,387,338✔





969












970



  if (pScanInfo->pFileDelData == NULL) {




7,387,338✔





971



    pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData));




2,789,597✔





972



    TSDB_CHECK_NULL(pScanInfo->pFileDelData, code, lino, _end, terrno);




2,789,597✔





973



  }










974












975



  for (int32_t k = 0; k < pBlock->numOfRecords; ++k) {




103,956,206✔





976



    code = tTombBlockGet(pBlock, k, &record);




99,792,996✔





977



    TSDB_CHECK_CODE(code, lino, _end);




99,792,996✔





978












979



    if (record.suid < pReader->info.suid) {




99,792,996✔





980



      continue;




×







981



    }










982












983



    if (record.suid > pReader->info.suid) {




99,792,996✔





984



      *pRet = BLK_CHECK_QUIT;




3,144,640✔





985



      goto _end;




3,144,640✔





986



    }










987












988



    if (uid < record.uid) {




96,648,356✔





989



      while ((*j) < numOfTables && pReader->status.uidList.tableUidList[*j] < record.uid) {




5,528,728✔





990



        (*j) += 1;




2,779,924✔





991



      }










992












993



      if ((*j) >= numOfTables) {




2,748,804✔





994



        *pRet = BLK_CHECK_QUIT;




79,488✔





995



        goto _end;




79,488✔





996



      }










997












998



      uid = pReader->status.uidList.tableUidList[*j];




2,669,316✔





999



      code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr);




2,669,316✔





1000



      TSDB_CHECK_CODE(code, lino, _end);




2,669,316✔





1001












1002



      if (pScanInfo->pFileDelData == NULL) {




2,669,316✔





1003



        pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData));




1,469,344✔





1004



        TSDB_CHECK_NULL(pScanInfo->pFileDelData, code, lino, _end, terrno);




1,469,344✔





1005



      }










1006



    }










1007












1008



    if (record.uid < uid) {




96,568,868✔





1009



      continue;




43,236,768✔





1010



    }










1011












1012



    TSDB_CHECK_CONDITION((record.suid == pReader->info.suid) && (uid == record.uid), code, lino, _end,




53,332,100✔





1013



                         TSDB_CODE_INTERNAL_ERROR);










1014












1015



    if (record.version <= pReader->info.verRange.maxVer) {




53,332,100✔





1016



      SDelData    delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey};




53,332,100✔





1017



      const void* px = taosArrayPush(pScanInfo->pFileDelData, &delData);




53,332,100✔





1018



      TSDB_CHECK_NULL(px, code, lino, _end, terrno);




53,332,100✔





1019



    }










1020



  }










1021












1022



  *pRet = BLK_CHECK_CONTINUE;




4,163,210✔





1023












1024



_end:




7,387,338✔





1025



  if (code != TSDB_CODE_SUCCESS) {




7,387,338✔





1026



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1027



  }










1028



  return code;




7,387,338✔





1029



}










1030












1031



// load tomb data API










1032



static int32_t doLoadTombDataFromTombBlk(const TTombBlkArray* pTombBlkArray, STsdbReader* pReader, void* pFileReader,




346,044,645✔





1033



                                         bool isFile) {










1034



  int32_t              code = TSDB_CODE_SUCCESS;




346,044,645✔





1035



  int32_t              lino = 0;




346,044,645✔





1036



  const STableUidList* pList = NULL;




346,044,645✔





1037



  int32_t              numOfTables = 0;




346,044,645✔





1038












1039



  TSDB_CHECK_NULL(pTombBlkArray, code, lino, _end, TSDB_CODE_INVALID_PARA);




346,044,645✔





1040



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




346,044,645✔





1041












1042



  pList = &pReader->status.uidList;




346,044,645✔





1043



  numOfTables = tSimpleHashGetSize(pReader->status.pTableMap);




346,036,219✔





1044












1045



  int32_t i = 0, j = 0;




346,285,707✔





1046



  while (i < pTombBlkArray->size && j < numOfTables) {




354,468,704✔





1047



    STombBlk* pTombBlk = &pTombBlkArray->data[i];




13,873,242✔





1048



    if (pTombBlk->maxTbid.suid < pReader->info.suid) {




13,873,242✔





1049



      i += 1;




2,934,661✔





1050



      continue;




2,934,661✔





1051



    }










1052












1053



    if (pTombBlk->minTbid.suid > pReader->info.suid) {




10,938,581✔





1054



      break;




2,349,236✔





1055



    }










1056












1057



    TSDB_CHECK_CONDITION(




8,589,345✔





1058



        (pTombBlk->minTbid.suid <= pReader->info.suid) && (pTombBlk->maxTbid.suid >= pReader->info.suid), code, lino,










1059



        _end, TSDB_CODE_INTERNAL_ERROR);










1060












1061



    if (pTombBlk->maxTbid.suid == pReader->info.suid && pTombBlk->maxTbid.uid < pList->tableUidList[0]) {




8,589,345✔





1062



      i += 1;




1,089,275✔





1063



      continue;




1,089,275✔





1064



    }










1065












1066



    if (pTombBlk->minTbid.suid == pReader->info.suid && pTombBlk->minTbid.uid > pList->tableUidList[numOfTables - 1]) {




7,500,070✔





1067



      break;




112,732✔





1068



    }










1069












1070



    STombBlock block = {0};




7,387,338✔





1071



    code = isFile ? tsdbDataFileReadTombBlock(pFileReader, &pTombBlkArray->data[i], &block)




3,277,248✔





1072



                  : tsdbSttFileReadTombBlock(pFileReader, &pTombBlkArray->data[i], &block);




7,387,338✔





1073



    TSDB_CHECK_CODE(code, lino, _end);




7,387,338✔





1074












1075



    ETombBlkCheckEnum ret = 0;




7,387,338✔





1076



    code = doCheckTombBlock(&block, pReader, numOfTables, &j, &ret);




7,387,338✔





1077












1078



    tTombBlockDestroy(&block);




7,387,338✔





1079



    TSDB_CHECK_CODE(code, lino, _end);




7,387,338✔





1080



    if (ret == BLK_CHECK_QUIT) {




7,387,338✔





1081



      break;




3,224,128✔





1082



    }










1083












1084



    i += 1;




4,163,210✔





1085



  }










1086












1087



_end:




346,245,324✔





1088



  if (code != TSDB_CODE_SUCCESS) {




346,270,668✔





1089



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1090



  }










1091



  return code;




346,270,668✔





1092



}










1093












1094



int32_t loadDataFileTombDataForAll(STsdbReader* pReader) {




382,736,918✔





1095



  int32_t              code = TSDB_CODE_SUCCESS;




382,736,918✔





1096



  int32_t              lino = 0;




382,736,918✔





1097



  const TTombBlkArray* pBlkArray = NULL;




382,736,918✔





1098












1099



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




382,827,011✔





1100












1101



  if ((pReader->status.pCurrentFileset == NULL) || (pReader->status.pCurrentFileset->farr[TSDB_FTYPE_TOMB] == NULL)) {




382,827,011✔





1102



    return TSDB_CODE_SUCCESS;




378,562,040✔





1103



  }










1104












1105



  code = tsdbDataFileReadTombBlk(pReader->pFileReader, &pBlkArray);




4,423,268✔





1106



  TSDB_CHECK_CODE(code, lino, _end);




4,423,268✔





1107












1108



  code = doLoadTombDataFromTombBlk(pBlkArray, pReader, pReader->pFileReader, true);




4,423,268✔





1109



  TSDB_CHECK_CODE(code, lino, _end);




4,423,268✔





1110












1111



_end:




4,423,268✔





1112



  if (code != TSDB_CODE_SUCCESS) {




4,423,268✔





1113



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1114



  }










1115



  return code;




4,423,268✔





1116



}










1117












1118



int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pLoadInfo) {




341,672,791✔





1119



  int32_t              code = TSDB_CODE_SUCCESS;




341,672,791✔





1120



  int32_t              lino = 0;




341,672,791✔





1121



  const TTombBlkArray* pBlkArray = NULL;




341,672,791✔





1122












1123



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




341,750,496✔





1124












1125



  code = tsdbSttFileReadTombBlk(pSttFileReader, &pBlkArray);




341,750,496✔





1126



  TSDB_CHECK_CODE(code, lino, _end);




341,818,076✔





1127












1128



  code = doLoadTombDataFromTombBlk(pBlkArray, pReader, pSttFileReader, false);




341,818,076✔





1129



  TSDB_CHECK_CODE(code, lino, _end);




341,789,526✔





1130












1131



_end:




341,789,526✔





1132



  if (code != TSDB_CODE_SUCCESS) {




341,822,636✔





1133



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1134



  }










1135



  return code;




341,822,636✔





1136



}










1137












1138



int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver) {




620,533,599✔





1139



  int32_t         code = TSDB_CODE_SUCCESS;




620,533,599✔





1140



  int32_t         lino = 0;




620,533,599✔





1141



  SArray*         pMemDelData = NULL;




620,533,599✔





1142



  const SDelData* p = NULL;




620,533,599✔





1143



  const void*     px = NULL;




620,533,599✔





1144












1145



  TSDB_CHECK_NULL(ppMemDelData, code, lino, _end, TSDB_CODE_INVALID_PARA);




620,533,599✔





1146












1147



  if (*ppMemDelData == NULL) {




620,533,599✔





1148



    *ppMemDelData = taosArrayInit(4, sizeof(SDelData));




617,776,265✔





1149



    TSDB_CHECK_NULL(*ppMemDelData, code, lino, _end, terrno);




617,361,314✔





1150



  }










1151












1152



  pMemDelData = *ppMemDelData;




620,438,707✔





1153












1154



  if (pMemTbData != NULL) {




620,343,935✔





1155



    taosRLockLatch(&pMemTbData->lock);




387,780,144✔





1156



    p = pMemTbData->pHead;




388,188,588✔





1157



    while (p) {




389,336,505✔





1158



      if (p->version <= ver) {




1,366,921✔





1159



        px = taosArrayPush(pMemDelData, p);




1,316,571✔





1160



        if (px == NULL) {




1,316,571✔





1161



          taosRUnLockLatch(&pMemTbData->lock);




×







1162



          TSDB_CHECK_NULL(px, code, lino, _end, terrno);




×







1163



        }










1164



      }










1165



      p = p->pNext;




1,366,921✔





1166



    }










1167



    taosRUnLockLatch(&pMemTbData->lock);




387,969,584✔





1168



  }










1169












1170



  if (piMemTbData != NULL) {




620,743,760✔





1171



    p = piMemTbData->pHead;




7,654,823✔





1172



    while (p) {




8,527,166✔





1173



      if (p->version <= ver) {




872,343✔





1174



        px = taosArrayPush(pMemDelData, p);




872,343✔





1175



        TSDB_CHECK_NULL(px, code, lino, _end, terrno);




872,343✔





1176



      }










1177



      p = p->pNext;




872,343✔





1178



    }










1179



  }










1180












1181



_end:




620,743,760✔





1182



  if (code != TSDB_CODE_SUCCESS) {




620,743,760✔





1183



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1184



  }










1185



  return code;




620,491,269✔





1186



}










1187












1188



int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo,




3,006✔





1189



                               TStatisBlkArray* pStatisBlkArray, uint64_t suid, const uint64_t* pUidList,










1190



                               int32_t numOfTables, int32_t* pNumOfRows) {










1191



  int32_t           code = TSDB_CODE_SUCCESS;




3,006✔





1192



  int32_t           lino = 0;




3,006✔





1193



  int32_t           num = 0;




3,006✔





1194



  int64_t           st = 0;




3,006✔





1195



  const SStatisBlk* p = NULL;




3,006✔





1196



  STbStatisBlock*   pStatisBlock = NULL;




3,006✔





1197












1198



  TSDB_CHECK_NULL(pSttFileReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




3,006✔





1199



  TSDB_CHECK_NULL(pBlockLoadInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




3,006✔





1200



  TSDB_CHECK_NULL(pStatisBlkArray, code, lino, _end, TSDB_CODE_INVALID_PARA);




3,006✔





1201



  TSDB_CHECK_NULL(pNumOfRows, code, lino, _end, TSDB_CODE_INVALID_PARA);




3,006✔





1202












1203



  *pNumOfRows = 0;




3,006✔





1204












1205



  if (TARRAY2_SIZE(pStatisBlkArray) <= 0) {




3,006✔





1206



    return code;




×







1207



  }










1208












1209



  int32_t i = 0;




3,006✔





1210



  while ((i < TARRAY2_SIZE(pStatisBlkArray)) && (pStatisBlkArray->data[i].maxTbid.suid < suid)) {




3,006✔





1211



    ++i;




×







1212



  }










1213












1214



  if (i >= TARRAY2_SIZE(pStatisBlkArray)) {




3,006✔





1215



    goto _end;




×







1216



  }










1217












1218



  p = &pStatisBlkArray->data[i];




3,006✔





1219



  pStatisBlock = taosMemoryCalloc(1, sizeof(STbStatisBlock));




3,006✔





1220



  TSDB_CHECK_NULL(pStatisBlock, code, lino, _end, terrno);




3,006✔





1221












1222



  code = tStatisBlockInit(pStatisBlock);




3,006✔





1223



  TSDB_CHECK_CODE(code, lino, _end);




3,006✔





1224












1225



  st = taosGetTimestampUs();




3,006✔





1226



  code = tsdbSttFileReadStatisBlock(pSttFileReader, p, pStatisBlock);




3,006✔





1227



  TSDB_CHECK_CODE(code, lino, _end);




3,006✔





1228












1229



  double el = (taosGetTimestampUs() - st) / 1000.0;




3,006✔





1230



  pBlockLoadInfo->cost.loadStatisBlocks += 1;




3,006✔





1231



  pBlockLoadInfo->cost.statisElapsedTime += el;




3,006✔





1232












1233



  int32_t index = 0;




3,006✔





1234



  while (index < pStatisBlock->numOfRecords && ((int64_t*)pStatisBlock->suids.data)[index] < suid) {




3,006✔





1235



    ++index;




×







1236



  }










1237












1238



  if (index >= pStatisBlock->numOfRecords) {




3,006✔





1239



    *pNumOfRows = num;




×







1240



    goto _end;




×







1241



  }










1242












1243



  int32_t j = index;




3,006✔





1244



  int32_t uidIndex = 0;




3,006✔





1245



  while (i < TARRAY2_SIZE(pStatisBlkArray) && uidIndex < numOfTables) {




5,491✔





1246



    p = &pStatisBlkArray->data[i];




3,039✔





1247



    if (p->minTbid.suid > suid) {




3,039✔





1248



      *pNumOfRows = num;




554✔





1249



      goto _end;




554✔





1250



    }










1251












1252



    uint64_t uid = pUidList[uidIndex];




2,485✔





1253












1254



    if (((int64_t*)pStatisBlock->uids.data)[j] == uid) {




2,485✔





1255



      num += ((int64_t*)pStatisBlock->counts.data)[j];




2,485✔





1256



      uidIndex += 1;




2,485✔





1257



      j += 1;




2,485✔





1258



      code = loadNextStatisticsBlock(pSttFileReader, pStatisBlock, pStatisBlkArray, pStatisBlock->numOfRecords, &i, &j);




2,485✔





1259



      TSDB_CHECK_CODE(code, lino, _end);




2,485✔





1260



    } else if (((int64_t*)pStatisBlock->uids.data)[j] < uid) {




×







1261



      j += 1;




×







1262



      code = loadNextStatisticsBlock(pSttFileReader, pStatisBlock, pStatisBlkArray, pStatisBlock->numOfRecords, &i, &j);




×







1263



      TSDB_CHECK_CODE(code, lino, _end);




×







1264



    } else {










1265



      uidIndex += 1;




×







1266



    }










1267



  }










1268












1269



  *pNumOfRows = num;




2,452✔





1270












1271



_end:




3,006✔





1272



  if (code != TSDB_CODE_SUCCESS) {




3,006✔





1273



    tsdbError("%s with %p failed at line %d since %s", __func__, pSttFileReader, lino, tstrerror(code));




×







1274



  }










1275



  if (pStatisBlock) {




3,006✔





1276



    tStatisBlockDestroy(pStatisBlock);




3,006✔





1277



    taosMemoryFreeClear(pStatisBlock);




3,006✔





1278



  }










1279



  return code;




3,006✔





1280



}










1281












1282



// load next stt statistics block










1283



static int32_t loadNextStatisticsBlock(SSttFileReader* pSttFileReader, STbStatisBlock* pStatisBlock,




2,485✔





1284



                                       const TStatisBlkArray* pStatisBlkArray, int32_t numOfRows, int32_t* i,










1285



                                       int32_t* j) {










1286



  int32_t code = TSDB_CODE_SUCCESS;




2,485✔





1287



  int32_t lino = 0;




2,485✔





1288












1289



  if ((*j) >= numOfRows) {




2,485✔





1290



    (*i) += 1;




2,452✔





1291



    (*j) = 0;




2,452✔





1292



    if ((*i) < TARRAY2_SIZE(pStatisBlkArray)) {




2,452✔





1293



      code = tsdbSttFileReadStatisBlock(pSttFileReader, &pStatisBlkArray->data[(*i)], pStatisBlock);




×







1294



      if (code != 0) {




×







1295



        tsdbError("%p failed to read statisBlock, code:%s", pSttFileReader, tstrerror(code));




×







1296



        TSDB_CHECK_CODE(code, lino, _end);




×







1297



      }










1298



    }










1299



  }










1300












1301



_end:




2,485✔





1302



  if (code != TSDB_CODE_SUCCESS) {




2,485✔





1303



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1304



  }










1305



  return code;




2,485✔





1306



}










1307












1308



int32_t doAdjustValidDataIters(SArray* pLDIterList, int32_t numOfFileObj) {




573,789,645✔





1309



  int32_t     code = TSDB_CODE_SUCCESS;




573,789,645✔





1310



  int32_t     lino = 0;




573,789,645✔





1311



  int32_t     size = 0;




573,789,645✔





1312



  int32_t     inc = 0;




573,789,645✔





1313



  SLDataIter* pIter = NULL;




573,789,645✔





1314












1315



  size = taosArrayGetSize(pLDIterList);




573,846,393✔





1316












1317



  if (size < numOfFileObj) {




573,827,738✔





1318



    inc = numOfFileObj - size;




341,680,620✔





1319



    for (int32_t k = 0; k < inc; ++k) {




683,580,259✔





1320



      pIter = taosMemoryCalloc(1, sizeof(SLDataIter));




341,824,871✔





1321



      TSDB_CHECK_NULL(pIter, code, lino, _end, terrno);




341,490,566✔





1322



      void* px = taosArrayPush(pLDIterList, &pIter);




341,899,639✔





1323



      if (px == NULL) {




341,899,639✔





1324



        TSDB_CHECK_NULL(px, code, lino, _end, terrno);




×







1325



        taosMemoryFreeClear(pIter);




×







1326



      }










1327



    }










1328



  } else if (size > numOfFileObj) {  // remove unused LDataIter




232,147,118✔





1329



    inc = size - numOfFileObj;




×







1330












1331



    for (int32_t i = 0; i < inc; ++i) {




×







1332



      pIter = taosArrayPop(pLDIterList);




×







1333



      destroyLDataIter(pIter);




×







1334



    }










1335



  }










1336












1337



_end:




573,901,494✔





1338



  if (code != TSDB_CODE_SUCCESS) {




573,767,997✔





1339



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1340



  }










1341



  return code;




573,767,997✔





1342



}










1343












1344



int32_t adjustSttDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet) {




543,924,139✔





1345



  int32_t  code = TSDB_CODE_SUCCESS;




543,924,139✔





1346



  int32_t  lino = 0;




543,924,139✔





1347



  int32_t  numOfLevels = 0;




543,924,139✔





1348



  SSttLvl* pSttLevel = NULL;




543,924,139✔





1349



  SArray*  pList = NULL;




543,924,139✔





1350












1351



  TSDB_CHECK_NULL(pFileSet, code, lino, _end, TSDB_CODE_INVALID_PARA);




544,122,446✔





1352












1353



  numOfLevels = pFileSet->lvlArr->size;




544,122,446✔





1354












1355



  // add the list/iter placeholder










1356



  while (taosArrayGetSize(pSttFileBlockIterArray) < numOfLevels) {




885,642,935✔





1357



    pList = taosArrayInit(4, POINTER_BYTES);




341,512,202✔





1358



    TSDB_CHECK_NULL(pList, code, lino, _end, terrno);




341,640,441✔





1359



    void* px = taosArrayPush(pSttFileBlockIterArray, &pList);




341,530,853✔





1360



    if (px == NULL) {




341,530,853✔





1361



      taosArrayDestroy(pList);




×







1362



      TSDB_CHECK_NULL(px, code, lino, _end, terrno);




×







1363



    }










1364



  }










1365












1366



  for (int32_t j = 0; j < numOfLevels; ++j) {




1,118,096,625✔





1367



    pSttLevel = pFileSet->lvlArr->data[j];




573,601,649✔





1368



    pList = taosArrayGetP(pSttFileBlockIterArray, j);




573,755,894✔





1369



    code = doAdjustValidDataIters(pList, TARRAY2_SIZE(pSttLevel->fobjArr));




573,871,339✔





1370



    TSDB_CHECK_CODE(code, lino, _end);




573,798,419✔





1371



  }










1372












1373



_end:




544,494,976✔





1374



  if (code != TSDB_CODE_SUCCESS) {




544,343,395✔





1375



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1376



  }










1377



  return code;




544,343,395✔





1378



}










1379












1380



int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArray, STsdb* pTsdb, SMergeTreeConf* pConf,




22,857✔





1381



                              const char* pstr) {










1382



  int32_t code = TSDB_CODE_SUCCESS;




22,857✔





1383



  int32_t lino = 0;




22,857✔





1384



  int32_t numOfRows = 0;




22,857✔





1385



  int32_t numOfLevels = 0;




22,857✔





1386












1387



  TSDB_CHECK_NULL(pFileSet, code, lino, _end, TSDB_CODE_INVALID_PARA);




22,857✔





1388












1389



  // no data exists, go to end










1390



  numOfLevels = pFileSet->lvlArr->size;




22,857✔





1391



  if (numOfLevels == 0) {




22,857✔





1392



    goto _end;




19,851✔





1393



  }










1394












1395



  // add the list/iter placeholder










1396



  code = adjustSttDataIters(pSttFileBlockIterArray, pFileSet);




3,006✔





1397



  TSDB_CHECK_CODE(code, lino, _end);




3,006✔





1398












1399



  for (int32_t j = 0; j < numOfLevels; ++j) {




6,012✔





1400



    SSttLvl* pSttLevel = pFileSet->lvlArr->data[j];




3,006✔





1401



    SArray*  pList = taosArrayGetP(pSttFileBlockIterArray, j);




3,006✔





1402












1403



    for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) {  // open all last file




6,012✔





1404



      SLDataIter* pIter = taosArrayGetP(pList, i);




3,006✔





1405












1406



      // open stt file reader if not opened yet










1407



      // if failed to open this stt file, ignore the error and try next one










1408



      if (pIter->pReader == NULL) {




3,006✔





1409



        SSttFileReaderConfig conf = {.tsdb = pTsdb, .szPage = pTsdb->pVnode->config.tsdbPageSize};




3,006✔





1410



        conf.file[0] = *pSttLevel->fobjArr->data[i]->f;




3,006✔





1411












1412



        const char* pName = pSttLevel->fobjArr->data[i]->fname;




3,006✔





1413



        code = tsdbSttFileReaderOpen(pName, &conf, &pIter->pReader);




3,006✔





1414



        if (code != TSDB_CODE_SUCCESS) {




3,006✔





1415



          tsdbError("open stt file reader error. file:%s, code %s, %s", pName, tstrerror(code), pstr);




×







1416



          continue;




×







1417



        }










1418



      }










1419












1420



      if (pIter->pBlockLoadInfo == NULL) {




3,006✔





1421



        code = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols, &pIter->pBlockLoadInfo);




3,006✔





1422



        if (code != TSDB_CODE_SUCCESS) {




3,006✔





1423



          tsdbError("failed to create block load info, code: out of memory, %s", pstr);




×







1424



          continue;




×







1425



        }










1426



      }










1427












1428



      // load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file










1429



      TStatisBlkArray* pStatisBlkArray = NULL;




3,006✔





1430



      code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray**)&pStatisBlkArray);




3,006✔





1431



      if (code != TSDB_CODE_SUCCESS) {




3,006✔





1432



        tsdbError("failed to load stt block statistics, code:%s, %s", tstrerror(code), pstr);




×







1433



        continue;




×







1434



      }










1435












1436



      // extract rows from each stt file one-by-one










1437



      STsdbReader* pReader = pConf->pReader;




3,006✔





1438



      int32_t      numOfTables = tSimpleHashGetSize(pReader->status.pTableMap);




3,006✔





1439



      uint64_t*    pUidList = pReader->status.uidList.tableUidList;




3,006✔





1440



      int32_t      n = 0;




3,006✔





1441



      code = getNumOfRowsInSttBlock(pIter->pReader, pIter->pBlockLoadInfo, pStatisBlkArray, pConf->suid, pUidList,




3,006✔





1442



                                    numOfTables, &n);










1443



      numOfRows += n;




3,006✔





1444



      if (code) {




3,006✔





1445



        tsdbError("%s failed to get rows in stt blocks, code:%s", pstr, tstrerror(code));




×







1446



      }










1447



    }










1448



  }










1449












1450



_end:




3,006✔





1451



  if (code != TSDB_CODE_SUCCESS) {




22,857✔





1452



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1453



  }










1454



  return numOfRows;




22,857✔





1455



}










1456












1457



static bool overlapHelper(const STimeWindow* pLeft, TSKEY minKey, TSKEY maxKey) {




4,397,737✔





1458



  return (pLeft != NULL) && (pLeft->ekey >= minKey) && (pLeft->skey <= maxKey);




4,397,737✔





1459



}










1460












1461



static bool overlapWithTimeWindow(STimeWindow* p1, STimeWindow* pQueryWindow, STableBlockScanInfo* pBlockScanInfo,




10,061,583✔





1462



                                  int32_t order) {










1463



  SIterInfo*   pMemIter = NULL;




10,061,583✔





1464



  SIterInfo*   pIMemIter = NULL;




10,061,583✔





1465



  STbData*     pTbData = NULL;




10,061,583✔





1466



  STimeWindow* pFileWin = NULL;




10,061,583✔





1467












1468



  if (p1 == NULL || pQueryWindow == NULL) {




10,061,583✔





1469



    return false;




×







1470



  }










1471












1472



  // overlap with query window










1473



  if (!(p1->skey >= pQueryWindow->skey && p1->ekey <= pQueryWindow->ekey)) {




10,066,933✔





1474



    return true;




121,188✔





1475



  }










1476












1477



  if (pBlockScanInfo == NULL) {




9,940,930✔





1478



    return false;




×







1479



  }










1480












1481



  pMemIter = &pBlockScanInfo->iter;




9,940,930✔





1482



  pIMemIter = &pBlockScanInfo->iiter;




9,939,284✔





1483












1484



  // overlap with mem data










1485



  if (pMemIter->hasVal) {




9,937,185✔





1486



    pTbData = pMemIter->iter->pTbData;




4,338,916✔





1487



    if (overlapHelper(p1, pTbData->minKey, pTbData->maxKey)) {




4,333,031✔





1488



      return true;




4,127,498✔





1489



    }










1490



  }










1491












1492



  // overlap with imem data










1493



  if (pIMemIter->hasVal) {




5,812,897✔





1494



    pTbData = pIMemIter->iter->pTbData;




57,303✔





1495



    if (overlapHelper(p1, pTbData->minKey, pTbData->maxKey)) {




56,768✔





1496



      return true;




32,610✔





1497



    }










1498



  }










1499












1500



  // overlap with data file block










1501



  pFileWin = &pBlockScanInfo->filesetWindow;




5,779,752✔





1502



  if ((taosArrayGetSize(pBlockScanInfo->pBlockIdxList) > 0) && overlapHelper(p1, pFileWin->skey, pFileWin->ekey)) {




5,780,822✔





1503



    return true;




×







1504



  }










1505












1506



  // overlap with deletion skyline










1507



  SBrinRecord record = {.firstKey = p1->skey, .lastKey = p1->ekey};




5,778,682✔





1508



  if (overlapWithDelSkylineWithoutVer(pBlockScanInfo, &record, order)) {




5,779,217✔





1509



    return true;




48,326✔





1510



  }










1511












1512



  return false;




5,728,710✔





1513



}










1514












1515



static int32_t sortUidComparFn(const void* p1, const void* p2) {




17,757,684✔





1516



  const SSttKeyRange* px1 = p1;




17,757,684✔





1517



  const SSttKeyRange* px2 = p2;




17,757,684✔





1518












1519



  int32_t ret = tRowKeyCompare(&px1->skey, &px2->skey);




17,757,684✔





1520



  return ret;




17,765,174✔





1521



}










1522












1523



bool isCleanSttBlock(SArray* pKeyRangeList, STimeWindow* pQueryWindow, STableBlockScanInfo* pScanInfo, int32_t order) {




12,905,666✔





1524



  int32_t       num = 0;




12,905,666✔





1525



  SSttKeyRange* pRange = NULL;




12,905,666✔





1526



  STimeWindow   w;




12,905,666✔





1527












1528



  num = taosArrayGetSize(pKeyRangeList);




12,909,912✔





1529



  if (num == 0) {




12,909,418✔





1530



    return false;




2,858,601✔





1531



  }










1532












1533



  // check if it overlap with del skyline










1534



  taosArraySort(pKeyRangeList, sortUidComparFn);




10,050,817✔





1535












1536



  pRange = taosArrayGet(pKeyRangeList, 0);




10,044,932✔





1537



  if (pRange == NULL) {




10,046,537✔





1538



    return false;




×







1539



  }










1540












1541



  w = (STimeWindow){.skey = pRange->skey.ts, .ekey = pRange->ekey.ts};




10,046,537✔





1542



  if (overlapWithTimeWindow(&w, pQueryWindow, pScanInfo, order)) {




10,049,747✔





1543



    return false;




4,328,017✔





1544



  }










1545












1546



  for (int32_t i = 0; i < num - 1; ++i) {




5,728,175✔





1547



    SSttKeyRange* p1 = taosArrayGet(pKeyRangeList, i);




3,296,635✔





1548



    SSttKeyRange* p2 = taosArrayGet(pKeyRangeList, i + 1);




3,298,775✔





1549



    if (p1 == NULL || p2 == NULL) {




3,298,775✔





1550



      return false;




×







1551



    }










1552












1553



    if (p1->ekey.ts >= p2->skey.ts) {




3,298,775✔





1554



      return false;




3,283,194✔





1555



    }










1556












1557



    w = (STimeWindow){.skey = p2->skey.ts, .ekey = p2->ekey.ts};




14,511✔





1558



    bool overlap = overlapWithTimeWindow(&w, pQueryWindow, pScanInfo, order);




14,511✔





1559



    if (overlap) {




14,511✔





1560



      return false;




×







1561



    }










1562



  }










1563












1564



  return true;




2,431,540✔





1565



}










1566












1567



static bool doCheckDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord,




138,920✔





1568



                                    int32_t startIndex) {










1569



  size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline);




138,920✔





1570












1571



  for (int32_t i = startIndex; i < num; i += 1) {




141,972✔





1572



    TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i);




141,972✔





1573



    if (p == NULL) {




141,972✔





1574



      return false;




×







1575



    }










1576












1577



    if (p->ts >= pRecord->firstKey.key.ts && p->ts <= pRecord->lastKey.key.ts) {




141,972✔





1578



      if (p->version >= pRecord->minVer) {




133,888✔





1579



        return true;




133,016✔





1580



      }










1581



    } else if (p->ts < pRecord->firstKey.key.ts) {  // p->ts < pBlock->minKey.ts




8,084✔





1582



      if (p->version >= pRecord->minVer) {




5,032✔





1583



        if (i < num - 1) {




2,852✔





1584



          TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1);




2,852✔





1585



          if (pnext == NULL) {




2,852✔





1586



            return false;




×







1587



          }










1588












1589



          if (pnext->ts >= pRecord->firstKey.key.ts) {




2,852✔





1590



            return true;




2,852✔





1591



          }










1592



        } else {  // it must be the last point










1593



          if (!(p->version == 0)) {




×







1594



            tsdbError("unexpected version:%" PRId64, p->version);




×







1595



          }










1596



        }










1597



      }










1598



    } else {  // (p->ts > pBlock->maxKey.ts) {










1599



      return false;




3,052✔





1600



    }










1601



  }










1602












1603



  return false;




×







1604



}










1605












1606



static bool doCheckDatablockOverlapWithoutVersion(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord,




48,326✔





1607



                                                  int32_t startIndex) {










1608



  size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline);




48,326✔





1609












1610



  for (int32_t i = startIndex; i < num; i += 1) {




48,326✔





1611



    TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i);




48,326✔





1612



    if (p == NULL) {




48,326✔





1613



      return false;




×







1614



    }










1615












1616



    if (p->ts >= pRecord->firstKey.key.ts && p->ts <= pRecord->lastKey.key.ts) {




48,326✔





1617



      return true;




48,326✔





1618



    } else if (p->ts < pRecord->firstKey.key.ts) {  // p->ts < pBlock->minKey.ts




×







1619



      if (i < num - 1) {




×







1620



        TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1);




×







1621



        if (pnext == NULL) {




×







1622



          return false;




×







1623



        }










1624












1625



        if (pnext->ts >= pRecord->firstKey.key.ts) {




×







1626



          return true;




×







1627



        }










1628



      }










1629



    } else {  // (p->ts > pBlock->maxKey.ts) {










1630



      return false;




×







1631



    }










1632



  }










1633












1634



  return false;




×







1635



}










1636












1637



bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, int32_t order) {




52,836,199✔





1638



  if (pBlockScanInfo->delSkyline == NULL || (taosArrayGetSize(pBlockScanInfo->delSkyline) == 0)) {




52,836,199✔





1639



    return false;




52,391,746✔





1640



  }










1641












1642



  // ts is not overlap










1643



  TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0);




449,919✔





1644



  TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline);




449,919✔





1645



  if (pFirst == NULL || pLast == NULL) {




449,919✔





1646



    return false;




×







1647



  }










1648












1649



  if (pRecord->firstKey.key.ts > pLast->ts || pRecord->lastKey.key.ts < pFirst->ts) {




449,919✔





1650



    return false;




310,999✔





1651



  }










1652












1653



  // version is not overlap










1654



  if (ASCENDING_TRAVERSE(order)) {




138,920✔





1655



    return doCheckDatablockOverlap(pBlockScanInfo, pRecord, pBlockScanInfo->fileDelIndex);




69,664✔





1656



  } else {










1657



    int32_t index = pBlockScanInfo->fileDelIndex;




69,256✔





1658



    while (1) {




61,476✔





1659



      TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, index);




130,732✔





1660



      if (p == NULL) {




130,732✔





1661



        return false;




×







1662



      }










1663












1664



      if (p->ts > pRecord->firstKey.key.ts && index > 0) {




130,732✔





1665



        index -= 1;




61,476✔





1666



      } else {  // find the first point that is smaller than the minKey.ts of dataBlock.










1667



        if (p->ts == pRecord->firstKey.key.ts && p->version < pRecord->maxVer && index > 0) {




69,256✔





1668



          index -= 1;




7,780✔





1669



        }










1670



        break;




69,256✔





1671



      }










1672



    }










1673












1674



    return doCheckDatablockOverlap(pBlockScanInfo, pRecord, index);




69,256✔





1675



  }










1676



}










1677












1678



bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, int32_t order) {




5,779,752✔





1679



  if (pBlockScanInfo->delSkyline == NULL || (taosArrayGetSize(pBlockScanInfo->delSkyline) == 0)) {




5,779,752✔





1680



    return false;




5,732,990✔





1681



  }










1682












1683



  // ts is not overlap










1684



  TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0);




48,326✔





1685



  TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline);




48,326✔





1686



  if (pFirst == NULL || pLast == NULL) {




48,326✔





1687



    return false;




×







1688



  }










1689












1690



  if (pRecord->firstKey.key.ts > pLast->ts || pRecord->lastKey.key.ts < pFirst->ts) {




48,326✔





1691



    return false;




×







1692



  }










1693












1694



  // version is not overlap










1695



  if (ASCENDING_TRAVERSE(order)) {




48,326✔





1696



    return doCheckDatablockOverlapWithoutVersion(pBlockScanInfo, pRecord, pBlockScanInfo->fileDelIndex);




48,326✔





1697



  } else {










1698



    int32_t index = pBlockScanInfo->fileDelIndex;




×







1699



    while (1) {




×







1700



      TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, index);




×







1701



      if (p == NULL) {




×







1702



        return false;




×







1703



      }










1704












1705



      if (p->ts > pRecord->firstKey.key.ts && index > 0) {




×







1706



        index -= 1;




×







1707



      } else {  // find the first point that is smaller than the minKey.ts of dataBlock.










1708



        if (p->ts == pRecord->firstKey.key.ts && index > 0) {




×







1709



          index -= 1;




×







1710



        }










1711



        break;




×







1712



      }










1713



    }










1714












1715



    return doCheckDatablockOverlapWithoutVersion(pBlockScanInfo, pRecord, index);




×







1716



  }










1717



}
























    

  • 

    •