taosdata / TDengine / #5059

coverage: 73.443% (+0.06%) from 73.387%
#5059

push

travis-ci

web-flow 
feat (TDgpt): Dynamic Model Synchronization Enhancements (#35344)

* refactor: do some internal refactor.

* fix: fix multiprocess sync issue.

* feat: add dynamic anomaly detection and forecasting services

* fix: log error message for undeploying model in exception handling

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* fix: handle undeploy when model exists only on disk

Agent-Logs-Url: https://github.com/taosdata/TDengine/sessions/286aafa0-c3ce-4c27-b803-2707571e9dc1

Co-authored-by: hjxilinx <8252296+hjxilinx@users.noreply.github.com>

* fix: guard dynamic registry concurrent access

Agent-Logs-Url: https://github.com/taosdata/TDengine/sessions/5e4db858-6458-40f4-ac28-d1b1b7f97c18

Co-authored-by: hjxilinx <8252296+hjxilinx@users.noreply.github.com>

* fix: tighten service list locking scope

Agent-Logs-Url: https://github.com/taosdata/TDengine/sessions/5e4db858-6458-40f4-ac28-d1b1b7f97c18

Co-authored-by: hjxilinx <8252296+hjxilinx@users.noreply.github.com>

* fix: restore prophet support and update tests per review feedback

Agent-Logs-Url: https://github.com/taosdata/TDengine/sessions/92298ae1-7da6-4d07-b20e-101c7cd0b26b

Co-authored-by: hjxilinx <8252296+hjxilinx@users.noreply.github.com>

* fix: improve test name and move copy inside lock scope

Agent-Logs-Url: https://github.com/taosdata/TDengine/sessions/92298ae1-7da6-4d07-b20e-101c7cd0b26b

Co-authored-by: hjxilinx <8252296+hjxilinx@users.noreply.github.com>

* Potential fix for pull request finding

Co-au... (continued)

281870 of 383795 relevant lines covered (73.44%)

135516561.93 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

86.25
/source/dnode/vnode/src/tsdb/tsdbReadUtil.c
1 
/*
2 
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3 
 *
4 
 * This program is free software: you can use, redistribute, and/or modify
5 
 * it under the terms of the GNU Affero General Public License, version 3
6 
 * or later ("AGPL"), as published by the Free Software Foundation.
7 
 *
8 
 * This program is distributed in the hope that it will be useful, but WITHOUT
9 
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 
 * FITNESS FOR A PARTICULAR PURPOSE.
11 
 *
12 
 * You should have received a copy of the GNU Affero General Public License
13 
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
14 
 */
15 









16



#include "tsdbReadUtil.h"










17



#include "tsdb.h"










18



#include "tsdbDataFileRW.h"










19



#include "tsdbFS2.h"










20



#include "tsdbMerge.h"










21



#include "tsdbUtil2.h"










22



#include "tsimplehash.h"










23












24



static bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord,










25



                                            int32_t order);










26












27



static int32_t initBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) {




312,188,792✔





28



  int32_t              code = TSDB_CODE_SUCCESS;




312,188,792✔





29



  int32_t              lino = 0;




312,188,792✔





30



  int32_t              num = 0;




312,188,792✔





31



  int32_t              remainder = 0;




312,188,792✔





32



  STableBlockScanInfo* p = NULL;




312,188,792✔





33



  const void*          px = NULL;




312,270,279✔





34












35



  TSDB_CHECK_CONDITION(pBuf && pBuf->numPerBucket > 0, code, lino, _end, TSDB_CODE_INVALID_PARA);




312,270,279✔





36



  TSDB_CHECK_CONDITION(numOfTables >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA);




312,216,978✔





37












38



  num = numOfTables / pBuf->numPerBucket;




312,216,978✔





39



  remainder = numOfTables % pBuf->numPerBucket;




312,242,624✔





40












41



  if (pBuf->pData == NULL) {




312,145,435✔





42



    pBuf->pData = taosArrayInit(num + 1, POINTER_BYTES);




312,101,859✔





43



    TSDB_CHECK_NULL(pBuf->pData, code, lino, _end, terrno);




312,172,469✔





44



  }










45












46



  for (int32_t i = 0; i < num; ++i) {




312,191,805✔





47



    p = taosMemoryCalloc(pBuf->numPerBucket, sizeof(STableBlockScanInfo));




34,084✔





48



    TSDB_CHECK_NULL(p, code, lino, _end, terrno);




34,084✔





49












50



    px = taosArrayPush(pBuf->pData, &p);




34,084✔





51



    TSDB_CHECK_NULL(px, code, lino, _end, terrno);




34,084✔





52



    p = NULL;




34,084✔





53



  }










54












55



  if (remainder > 0) {




312,157,721✔





56



    p = taosMemoryCalloc(remainder, sizeof(STableBlockScanInfo));




296,247,872✔





57



    TSDB_CHECK_NULL(p, code, lino, _end, terrno);




296,218,072✔





58












59



    px = taosArrayPush(pBuf->pData, &p);




296,218,072✔





60



    TSDB_CHECK_NULL(px, code, lino, _end, terrno);




296,381,715✔





61



    p = NULL;




296,381,715✔





62



  }










63












64



  pBuf->numOfTables = numOfTables;




312,291,564✔





65












66



_end:




312,338,572✔





67



  if (code != TSDB_CODE_SUCCESS) {




312,344,417✔





68



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







69



  }










70



  if (p) {




312,242,758✔





71



    taosMemoryFreeClear(p);




×







72



  }










73



  return code;




312,242,758✔





74



}










75












76



int32_t uidComparFunc(const void* p1, const void* p2) {




1,677,771,429✔





77



  uint64_t pu1 = *(const uint64_t*)p1;




1,677,771,429✔





78



  uint64_t pu2 = *(const uint64_t*)p2;




1,677,810,629✔





79



  if (pu1 == pu2) {




1,677,892,423✔





80



    return 0;




×







81



  } else {










82



    return (pu1 < pu2) ? -1 : 1;




1,677,892,423✔





83



  }










84



}










85












86



int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) {




3,692✔





87



  int32_t              code = TSDB_CODE_SUCCESS;




3,692✔





88



  int32_t              lino = 0;




3,692✔





89



  int32_t              num = 0;




3,692✔





90



  int32_t              remainder = 0;




3,692✔





91



  STableBlockScanInfo* p = NULL;




3,692✔





92



  const void*          px = NULL;




3,692✔





93












94



  TSDB_CHECK_CONDITION(pBuf && pBuf->numPerBucket > 0 && pBuf->numOfTables >= 0, code, lino, _end,




3,692✔





95



                       TSDB_CODE_INVALID_PARA);










96



  TSDB_CHECK_CONDITION(numOfTables >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA);




3,692✔





97












98



  if (numOfTables <= pBuf->numOfTables) {




3,692✔





99



    goto _end;




×







100



  }










101












102



  remainder = pBuf->numOfTables % pBuf->numPerBucket;




3,692✔





103



  if (remainder > 0) {




3,692✔





104



    TSDB_CHECK_CONDITION(taosArrayGetSize(pBuf->pData) > 0, code, lino, _end, TSDB_CODE_INVALID_PARA);




3,692✔





105



    px = taosArrayPop(pBuf->pData);




3,692✔





106



    TSDB_CHECK_NULL(px, code, lino, _end, TSDB_CODE_INVALID_PARA);




3,692✔





107



    p = *(STableBlockScanInfo**)px;




3,692✔





108



    taosMemoryFreeClear(p);




3,692✔





109



    pBuf->numOfTables -= remainder;




3,692✔





110



  }










111












112



  num = (numOfTables - pBuf->numOfTables) / pBuf->numPerBucket;




3,692✔





113



  remainder = (numOfTables - pBuf->numOfTables) % pBuf->numPerBucket;




3,692✔





114












115



  if (pBuf->pData == NULL) {




3,692✔





116



    pBuf->pData = taosArrayInit(num + 1, POINTER_BYTES);




×







117



    TSDB_CHECK_NULL(pBuf->pData, code, lino, _end, terrno);




×







118



  }










119












120



  for (int32_t i = 0; i < num; ++i) {




3,692✔





121



    p = taosMemoryCalloc(pBuf->numPerBucket, sizeof(STableBlockScanInfo));




×







122



    TSDB_CHECK_NULL(p, code, lino, _end, terrno);




×







123












124



    px = taosArrayPush(pBuf->pData, &p);




×







125



    TSDB_CHECK_NULL(px, code, lino, _end, terrno);




×







126



    p = NULL;




×







127



  }










128












129



  if (remainder > 0) {




3,692✔





130



    p = taosMemoryCalloc(remainder, sizeof(STableBlockScanInfo));




3,692✔





131



    TSDB_CHECK_NULL(p, code, lino, _end, terrno);




3,692✔





132












133



    px = taosArrayPush(pBuf->pData, &p);




3,692✔





134



    TSDB_CHECK_NULL(px, code, lino, _end, terrno);




3,692✔





135



    p = NULL;




3,692✔





136



  }










137












138



  pBuf->numOfTables = numOfTables;




3,692✔





139












140



_end:




3,692✔





141



  if (code != TSDB_CODE_SUCCESS) {




3,692✔





142



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







143



  }










144



  if (p) {




3,692✔





145



    taosMemoryFreeClear(p);




×







146



  }










147



  return code;




3,692✔





148



}










149












150



void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf) {




322,246,942✔





151



  if (pBuf == NULL) return;




322,246,942✔





152



  if (pBuf->pData != NULL) {




322,246,942✔





153



    taosArrayDestroyP(pBuf->pData, NULL);




312,390,591✔





154



    pBuf->pData = NULL;




312,297,233✔





155



  }










156



}










157












158



int32_t getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index, STableBlockScanInfo** pInfo) {




772,545,178✔





159



  int32_t               code = TSDB_CODE_SUCCESS;




772,545,178✔





160



  int32_t               lino = 0;




772,545,178✔





161



  int32_t               bucketIndex = 0;




772,545,178✔





162



  STableBlockScanInfo** pBucket = NULL;




772,545,178✔





163












164



  TSDB_CHECK_CONDITION(pBuf && pBuf->numPerBucket > 0, code, lino, _end, TSDB_CODE_INVALID_PARA);




772,545,178✔





165



  TSDB_CHECK_CONDITION(index >= 0 && index < pBuf->numOfTables, code, lino, _end, TSDB_CODE_INVALID_PARA);




772,534,874✔





166



  TSDB_CHECK_NULL(pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




772,451,700✔





167












168



  *pInfo = NULL;




772,451,700✔





169












170



  bucketIndex = index / pBuf->numPerBucket;




772,576,601✔





171



  pBucket = taosArrayGet(pBuf->pData, bucketIndex);




772,654,429✔





172



  TSDB_CHECK_NULL(pBucket, code, lino, _end, terrno);




772,417,598✔





173












174



  *pInfo = (*pBucket) + (index % pBuf->numPerBucket);




772,417,598✔





175












176



_end:




772,520,400✔





177



  if (code != TSDB_CODE_SUCCESS) {




772,520,400✔





178



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







179



  }










180



  return code;




772,614,824✔





181



}










182












183



int32_t getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, STableBlockScanInfo** pInfo, const char* id) {




544,154,048✔





184



  int32_t               code = TSDB_CODE_SUCCESS;




544,154,048✔





185



  int32_t               lino = 0;




544,154,048✔





186



  STableBlockScanInfo** pVal = NULL;




544,154,048✔





187












188



  TSDB_CHECK_NULL(pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




544,154,048✔





189



  TSDB_CHECK_NULL(id, code, lino, _end, TSDB_CODE_INVALID_PARA);




544,154,048✔





190












191



  pVal = (STableBlockScanInfo**)tSimpleHashGet(pTableMap, &uid, sizeof(uid));




544,154,048✔





192



  if (pVal == NULL) {




544,336,636✔





193



    int32_t size = tSimpleHashGetSize(pTableMap);




×







194



    tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id);




×







195



    code = TSDB_CODE_INVALID_PARA;




×







196



    TSDB_CHECK_CODE(code, lino, _end);




×







197



  }










198



  *pInfo = *pVal;




544,336,636✔





199



  TSDB_CHECK_NULL(*pInfo, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);




544,321,670✔





200












201



_end:




544,325,414✔





202



  if (code != TSDB_CODE_SUCCESS) {




544,325,414✔





203



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







204



  }










205



  return code;




544,236,086✔





206



}










207












208



int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc) {




2,147,483,647✔





209



  int32_t code = TSDB_CODE_SUCCESS;




2,147,483,647✔





210



  int32_t lino = 0;




2,147,483,647✔





211












212



  TSDB_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





213












214



  pKey->numOfPKs = numOfPks;




2,147,483,647✔





215



  pKey->ts = ts;




2,147,483,647✔





216












217



  if (numOfPks > 0) {




2,147,483,647✔





218



    pKey->pks[0].type = type;




130,563,727✔





219












220



    if (IS_NUMERIC_TYPE(type)) {




130,548,962✔





221



      if (asc) {




90,669,459✔





222



        switch (type) {




57,370,407✔





223



          case TSDB_DATA_TYPE_BIGINT: {




14,203,476✔





224



            VALUE_SET_TRIVIAL_DATUM(pKey->pks, INT64_MIN);




14,203,476✔





225



            break;




14,206,332✔





226



          }










227



          case TSDB_DATA_TYPE_INT: {




14,983,226✔





228



            int32_t min = INT32_MIN;




14,983,226✔





229



            valueSetDatum(pKey->pks, type, &min, tDataTypes[type].bytes);




14,983,918✔





230



            break;




14,990,344✔





231



          }










232



          case TSDB_DATA_TYPE_SMALLINT: {




×







233



            int16_t min = INT16_MIN;




×







234



            valueSetDatum(pKey->pks, type, &min, tDataTypes[type].bytes);




×







235



            break;




×







236



          }










237



          case TSDB_DATA_TYPE_TINYINT: {




×







238



            int8_t min = INT8_MIN;




×







239



            valueSetDatum(pKey->pks, type, &min, tDataTypes[type].bytes);




×







240



            break;




×







241



          }










242



          case TSDB_DATA_TYPE_UTINYINT:




28,165,710✔





243



          case TSDB_DATA_TYPE_USMALLINT:










244



          case TSDB_DATA_TYPE_UINT:










245



          case TSDB_DATA_TYPE_UBIGINT: {










246



            VALUE_SET_TRIVIAL_DATUM(pKey->pks, 0);




28,165,710✔





247



            break;




28,176,554✔





248



          }










249



          default:




17,995✔





250



            code = TSDB_CODE_INVALID_PARA;




17,995✔





251



            TSDB_CHECK_CODE(code, lino, _end);




17,995✔





252



        }










253



      } else {










254



        switch (type) {




33,299,052✔





255



          case TSDB_DATA_TYPE_BIGINT:




8,200,180✔





256



            VALUE_SET_TRIVIAL_DATUM(pKey->pks, INT64_MAX);




8,200,180✔





257



            break;




8,201,564✔





258



          case TSDB_DATA_TYPE_INT:




8,585,950✔





259



            VALUE_SET_TRIVIAL_DATUM(pKey->pks, INT32_MAX);




8,585,950✔





260



            break;




8,586,642✔





261



          case TSDB_DATA_TYPE_SMALLINT:




×







262



            VALUE_SET_TRIVIAL_DATUM(pKey->pks, INT16_MAX);




×







263



            break;




×







264



          case TSDB_DATA_TYPE_TINYINT:




×







265



            VALUE_SET_TRIVIAL_DATUM(pKey->pks, INT8_MAX);




×







266



            break;




×







267



          case TSDB_DATA_TYPE_UBIGINT:




8,194,420✔





268



            VALUE_SET_TRIVIAL_DATUM(pKey->pks, UINT64_MAX);




8,194,420✔





269



            break;




8,194,420✔





270



          case TSDB_DATA_TYPE_UINT:




8,319,864✔





271



            VALUE_SET_TRIVIAL_DATUM(pKey->pks, UINT32_MAX);




8,319,864✔





272



            break;




8,321,280✔





273



          case TSDB_DATA_TYPE_USMALLINT:




×







274



            VALUE_SET_TRIVIAL_DATUM(pKey->pks, UINT16_MAX);




×







275



            break;




×







276



          case TSDB_DATA_TYPE_UTINYINT:




×







277



            VALUE_SET_TRIVIAL_DATUM(pKey->pks, UINT8_MAX);




×







278



            break;




×







279



          default:




22✔





280



            code = TSDB_CODE_INVALID_PARA;




22✔





281



            TSDB_CHECK_CODE(code, lino, _end);




22✔





282



        }










283



      }










284



    } else {










285



      pKey->pks[0].nData = 0;




39,879,503✔





286












287



      if (pKey->pks[0].pData == NULL) {




39,888,129✔





288



        pKey->pks[0].pData = taosMemoryCalloc(1, len);




39,887,385✔





289



        TSDB_CHECK_NULL(pKey->pks[0].pData, code, lino, _end, terrno);




39,750,757✔





290



      }










291












292



      if (!asc) {




39,772,825✔





293



        pKey->numOfPKs = 2;




16,535,558✔





294



      }










295



    }










296



  }










297












298



_end:




2,147,483,647✔





299



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





300



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







301



  }










302



  return code;




2,147,483,647✔





303



}










304












305



void clearRowKey(SRowKey* pKey) {




2,147,483,647✔





306



  if (pKey == NULL || pKey->numOfPKs == 0 || (!IS_VAR_DATA_TYPE(pKey->pks[0].type))) {




2,147,483,647✔





307



    return;




2,147,483,647✔





308



  }










309



  taosMemoryFreeClear(pKey->pks[0].pData);




2,974,688✔





310



  taosMemoryFreeClear(pKey->pks[1].pData);




46,402,145✔





311



}










312












313



static int32_t initLastProcKey(STableBlockScanInfo* pScanInfo, const STsdbReader* pReader) {




772,564,301✔





314



  int32_t  code = TSDB_CODE_SUCCESS;




772,564,301✔





315



  int32_t  lino = 0;




772,564,301✔





316



  int32_t  numOfPks = 0;




772,564,301✔





317



  int32_t  type = 0;




772,564,301✔





318



  int32_t  bytes = 0;




772,564,301✔





319



  bool     asc = false;




772,564,301✔





320



  SRowKey* pRowKey = NULL;




772,564,301✔





321












322



  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




772,564,301✔





323



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




772,564,301✔





324












325



  numOfPks = pReader->suppInfo.numOfPks;




772,564,301✔





326



  asc = ASCENDING_TRAVERSE(pReader->info.order);




772,546,114✔





327



  type = pReader->suppInfo.pk.type;




772,507,113✔





328



  bytes = pReader->suppInfo.pk.bytes;




772,576,207✔





329












330



  pRowKey = &pScanInfo->lastProcKey;




772,684,631✔





331



  if (asc) {




772,595,040✔





332



    int64_t skey = pReader->info.window.skey;




712,314,700✔





333



    int64_t ts = (skey > INT64_MIN) ? (skey - 1) : skey;




712,088,111✔





334












335



    code = initRowKey(pRowKey, ts, numOfPks, type, bytes, asc);




712,088,111✔





336



    TSDB_CHECK_CODE(code, lino, _end);




712,186,042✔





337












338



    code = initRowKey(&pScanInfo->sttKeyInfo.nextProcKey, skey, numOfPks, type, bytes, asc);




712,186,042✔





339



    TSDB_CHECK_CODE(code, lino, _end);




712,221,065✔





340



  } else {










341



    int64_t ekey = pReader->info.window.ekey;




60,280,340✔





342



    int64_t ts = (ekey < INT64_MAX) ? (ekey + 1) : ekey;




60,294,978✔





343












344



    code = initRowKey(pRowKey, ts, numOfPks, type, bytes, asc);




60,294,978✔





345



    TSDB_CHECK_CODE(code, lino, _end);




60,286,160✔





346












347



    code = initRowKey(&pScanInfo->sttKeyInfo.nextProcKey, ekey, numOfPks, type, bytes, asc);




60,286,160✔





348



    TSDB_CHECK_CODE(code, lino, _end);




60,291,456✔





349



  }










350












351



  code = initRowKey(&pScanInfo->sttRange.skey, INT64_MAX, numOfPks, type, bytes, asc);




772,512,521✔





352



  TSDB_CHECK_CODE(code, lino, _end);




772,567,566✔





353












354



  code = initRowKey(&pScanInfo->sttRange.ekey, INT64_MIN, numOfPks, type, bytes, asc);




772,567,566✔





355



  TSDB_CHECK_CODE(code, lino, _end);




772,516,036✔





356












357



_end:




772,516,036✔





358



  if (code != TSDB_CODE_SUCCESS) {




772,516,036✔





359



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







360



  }










361



  return code;




772,410,641✔





362



}










363












364



int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSHashObj* pTableMap,




772,591,809✔





365



                               STsdbReader* pReader) {










366



  int32_t code = TSDB_CODE_SUCCESS;




772,591,809✔





367



  int32_t lino = 0;




772,591,809✔





368












369



  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




772,591,809✔





370



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




772,591,809✔





371












372



  pScanInfo->uid = uid;




772,591,809✔





373



  INIT_KEYRANGE(&pScanInfo->sttRange);




772,609,715✔





374



  INIT_TIMEWINDOW(&pScanInfo->filesetWindow);




772,438,380✔





375












376



  pScanInfo->cleanSttBlocks = false;




772,604,865✔





377



  pScanInfo->sttBlockReturned = false;




772,433,037✔





378












379



  code = initLastProcKey(pScanInfo, pReader);




772,493,777✔





380



  TSDB_CHECK_CODE(code, lino, _end);




772,366,225✔





381












382



  pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT;




772,366,225✔





383



  code = tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES);




772,476,759✔





384



  TSDB_CHECK_CODE(code, lino, _end);




772,483,923✔





385












386



  tsdbTrace("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pReader, pScanInfo->uid,




772,483,923✔





387



            pScanInfo->lastProcKey.ts, pReader->idStr);










388












389



_end:




759,047,490✔





390



  if (code != TSDB_CODE_SUCCESS) {




772,524,527✔





391



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







392



  }










393



  return code;




772,569,752✔





394



}










395












396



// NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model










397



int32_t createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList,




312,167,515✔





398



                                STableUidList* pUidList, int32_t numOfTables, SSHashObj** pHashObj) {










399



  int32_t    code = TSDB_CODE_SUCCESS;




312,167,515✔





400



  int32_t    lino = 0;




312,167,515✔





401



  SSHashObj* pTableMap = NULL;




312,167,515✔





402



  int64_t    st = 0;




312,167,515✔





403












404



  TSDB_CHECK_NULL(pUidList, code, lino, _end, TSDB_CODE_INVALID_PARA);




312,167,515✔





405



  TSDB_CHECK_NULL(pHashObj, code, lino, _end, TSDB_CODE_INVALID_PARA);




312,167,515✔





406












407



  *pHashObj = NULL;




312,167,515✔





408












409



  // allocate buffer in order to load data blocks from file










410



  // todo use simple hash instead, optimize the memory consumption










411



  pTableMap = tSimpleHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));




312,153,854✔





412



  TSDB_CHECK_NULL(pTableMap, code, lino, _end, terrno);




312,132,011✔





413












414



  st = taosGetTimestampUs();




312,272,468✔





415



  code = initBlockScanInfoBuf(pBuf, numOfTables);




312,272,468✔





416



  if (code != TSDB_CODE_SUCCESS) {




312,221,040✔





417



    tSimpleHashCleanup(pTableMap);




×







418



    TSDB_CHECK_CODE(code, lino, _end);




×







419



  }










420












421



  pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t));




312,221,040✔





422



  if (pUidList->tableUidList == NULL) {




312,116,478✔





423



    tSimpleHashCleanup(pTableMap);




×







424



    TSDB_CHECK_NULL(pUidList->tableUidList, code, lino, _end, terrno);




×







425



  }










426












427



  pUidList->currentIndex = 0;




312,238,219✔





428












429



  for (int32_t j = 0; j < numOfTables; ++j) {




1,060,614,462✔





430



    pUidList->tableUidList[j] = idList[j].uid;




748,277,675✔





431












432



    STableBlockScanInfo* pScanInfo = NULL;




748,344,020✔





433



    code = getPosInBlockInfoBuf(pBuf, j, &pScanInfo);




748,290,668✔





434



    if (code != TSDB_CODE_SUCCESS) {




748,369,961✔





435



      lino = __LINE__;




×







436



      break;




×







437



    }










438












439



    code = initTableBlockScanInfo(pScanInfo, idList[j].uid, pTableMap, pTsdbReader);




748,369,961✔





440



    if (code != TSDB_CODE_SUCCESS) {




748,267,863✔





441



      lino = __LINE__;




×







442



      break;




×







443



    }










444



  }










445












446



  taosSort(pUidList->tableUidList, numOfTables, sizeof(uint64_t), uidComparFunc);




312,340,981✔





447












448



  pTsdbReader->cost.createScanInfoList = (taosGetTimestampUs() - st) / 1000.0;




312,337,549✔





449



  tsdbDebug("%p create %d tables scan-info, size:%.2f Kb, elapsed time:%.2f ms, %s", pTsdbReader, numOfTables,




312,202,246✔





450



            (sizeof(STableBlockScanInfo) * numOfTables) / 1024.0, pTsdbReader->cost.createScanInfoList,










451



            pTsdbReader->idStr);










452












453



  *pHashObj = pTableMap;




312,202,470✔





454












455



_end:




312,195,431✔





456



  if (code != TSDB_CODE_SUCCESS) {




312,195,431✔





457



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







458



  }










459



  return code;




312,199,699✔





460



}










461












462



void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) {




30,108,195✔





463



  STableBlockScanInfo** p = NULL;




30,108,195✔





464



  int32_t               iter = 0;




30,108,195✔





465












466



  while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) {




60,352,948✔





467



    STableBlockScanInfo* pInfo = *p;




30,274,536✔





468



    if (pInfo == NULL) {




30,275,876✔





469



      continue;




×







470



    }










471












472



    pInfo->iterInit = false;




30,275,876✔





473



    pInfo->iter.hasVal = false;




30,273,392✔





474



    pInfo->iiter.hasVal = false;




30,270,515✔





475












476



    if (pInfo->iter.iter != NULL) {




30,259,652✔





477



      pInfo->iter.iter = tsdbTbDataIterDestroy(pInfo->iter.iter);




5,046,169✔





478



    }










479












480



    if (pInfo->iiter.iter != NULL) {




30,262,856✔





481



      pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter);




549✔





482



    }










483












484



    taosArrayDestroy(pInfo->delSkyline);




30,267,104✔





485



    pInfo->delSkyline = NULL;




30,263,991✔





486



    pInfo->lastProcKey.ts = ts;




30,270,902✔





487



    // todo check the nextProcKey info










488



    if ((ts == INT64_MAX && step > 0) || (ts == INT64_MIN && step < 0)) {




30,267,640✔





489



      tsdbError("%s nextProcKey ts overflow, ts:%" PRId64 ", step:%" PRId32,




8✔





490



                __func__, ts, step);










491



    } else {










492



      pInfo->sttKeyInfo.nextProcKey.ts = ts + step;




30,267,632✔





493



    }










494



  }










495



}




30,089,276✔





496












497



// keep the lastProcKey/sttRange/nextProcKey info










498



void clearBlockScanInfoLoadInfo(STableBlockScanInfo* p) {




772,838,898✔





499



  if (p == NULL) {




772,838,898✔





500



    return;




×







501



  }










502












503



  p->iterInit = false;




772,838,898✔





504



  p->iter.hasVal = false;




772,870,142✔





505



  p->iiter.hasVal = false;




772,865,583✔





506



  p->sttKeyInfo.status = STT_FILE_READER_UNINIT;




772,754,865✔





507












508



  if (p->iter.iter != NULL) {




772,795,304✔





509



    p->iter.iter = tsdbTbDataIterDestroy(p->iter.iter);




457,413,639✔





510



  }










511












512



  if (p->iiter.iter != NULL) {




772,727,812✔





513



    p->iiter.iter = tsdbTbDataIterDestroy(p->iiter.iter);




5,125,765✔





514



  }










515












516



  taosArrayDestroy(p->delSkyline);




772,823,069✔





517



  p->delSkyline = NULL;




772,809,260✔





518



  taosArrayDestroy(p->pBlockList);




772,833,278✔





519



  p->pBlockList = NULL;




772,761,046✔





520



  taosArrayDestroy(p->pBlockIdxList);




772,757,488✔





521



  p->pBlockIdxList = NULL;




772,765,851✔





522



  taosArrayDestroy(p->pMemDelData);




772,775,223✔





523



  p->pMemDelData = NULL;




772,770,681✔





524



  taosArrayDestroy(p->pFileDelData);




772,785,144✔





525



  p->pFileDelData = NULL;




772,781,485✔





526



}










527












528



void clearBlockScanInfo(STableBlockScanInfo* p) {




772,843,427✔





529



  if (p == NULL) {




772,843,427✔





530



    return;




×







531



  }










532












533



  clearBlockScanInfoLoadInfo(p);




772,843,427✔





534












535



  clearRowKey(&p->lastProcKey);




772,793,910✔





536



  clearRowKey(&p->sttRange.skey);




772,769,044✔





537



  clearRowKey(&p->sttRange.ekey);




772,789,465✔





538



  clearRowKey(&p->sttKeyInfo.nextProcKey);




772,780,893✔





539



}










540












541



void destroyAllBlockScanInfo(SSHashObj** pTableMap) {




312,385,651✔





542



  STableBlockScanInfo** p = NULL;




312,385,651✔





543



  int32_t               iter = 0;




312,385,651✔





544












545



  if (pTableMap == NULL || *pTableMap == NULL) {




312,401,474✔





546



    return;




×







547



  }










548












549



  while ((p = tSimpleHashIterate(*pTableMap, p, &iter)) != NULL) {




1,060,916,287✔





550



    clearBlockScanInfo(*p);




748,519,012✔





551



  }










552












553



  tSimpleHashCleanup(*pTableMap);




312,352,362✔





554



  *pTableMap = NULL;




312,344,186✔





555



}










556












557



static void doCleanupInfoForNextFileset(STableBlockScanInfo* pScanInfo) {




1,116,574,865✔





558



  if (pScanInfo == NULL) {




1,116,574,865✔





559



    return;




×







560



  }










561



  // reset the index in last block when handing a new file










562



  taosArrayClear(pScanInfo->pBlockList);




1,116,574,865✔





563



  taosArrayClear(pScanInfo->pBlockIdxList);




1,116,724,479✔





564



  taosArrayClear(pScanInfo->pFileDelData);  // del data from each file set




1,116,723,506✔





565



  pScanInfo->cleanSttBlocks = false;




1,116,842,821✔





566



  pScanInfo->numOfRowsInStt = 0;




1,116,758,035✔





567



  pScanInfo->sttBlockReturned = false;




1,116,774,001✔





568



  INIT_KEYRANGE(&pScanInfo->sttRange);




1,116,836,575✔





569



  INIT_TIMEWINDOW(&pScanInfo->filesetWindow);




1,116,731,862✔





570



  pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT;




1,116,847,049✔





571



}










572












573



void cleanupInfoForNextFileset(SSHashObj* pTableMap) {




592,412,893✔





574



  STableBlockScanInfo** p = NULL;




592,412,893✔





575



  int32_t               iter = 0;




592,412,893✔





576












577



  while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) {




1,709,252,214✔





578



    doCleanupInfoForNextFileset(*p);




1,116,710,031✔





579



  }










580



}




592,733,630✔





581












582



// brin records iterator










583



void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList) {




592,607,626✔





584



  if (pIter == NULL) {




592,607,626✔





585



    return;




×







586



  }










587












588



  (void)memset(&pIter->block, 0, sizeof(SBrinBlock));




592,607,626✔





589



  (void)memset(&pIter->record, 0, sizeof(SBrinRecord));




592,757,585✔





590



  pIter->blockIndex = -1;




592,654,338✔





591



  pIter->recordIndex = -1;




592,796,383✔





592












593



  pIter->pReader = pReader;




592,736,152✔





594



  pIter->pBrinBlockList = pList;




592,684,848✔





595



}










596












597



int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord) {




1,455,048,958✔





598



  int32_t code = TSDB_CODE_SUCCESS;




1,455,048,958✔





599



  int32_t lino = 0;




1,455,048,958✔





600












601



  TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,455,048,958✔





602



  TSDB_CHECK_NULL(pRecord, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,455,048,958✔





603












604



  *pRecord = NULL;




1,455,048,958✔





605












606



  if (pIter->blockIndex == -1 || (pIter->recordIndex + 1) >= pIter->block.numOfRecords) {




1,455,196,388✔





607



    pIter->blockIndex += 1;




609,552,098✔





608



    if (pIter->blockIndex >= taosArrayGetSize(pIter->pBrinBlockList)) {




609,525,391✔





609



      goto _end;




585,024,546✔





610



    }










611












612



    pIter->pCurrentBlk = taosArrayGet(pIter->pBrinBlockList, pIter->blockIndex);




24,557,351✔





613



    TSDB_CHECK_NULL(pIter->pCurrentBlk, code, lino, _end, terrno);




24,555,543✔





614












615



    tBrinBlockClear(&pIter->block);




24,542,682✔





616



    TSDB_CHECK_NULL(pIter->pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




24,544,146✔





617



    code = tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block);




24,544,146✔





618



    if (code != TSDB_CODE_SUCCESS) {




24,561,498✔





619



      tsdbError("failed to read brinBlock from file, code:%s", tstrerror(code));




×







620



      TSDB_CHECK_CODE(code, lino, _end);




×







621



    }










622












623



    pIter->recordIndex = -1;




24,561,498✔





624



  }










625












626



  pIter->recordIndex += 1;




870,804,124✔





627



  code = tBrinBlockGet(&pIter->block, pIter->recordIndex, &pIter->record);




870,854,421✔





628



  *pRecord = &pIter->record;




870,058,116✔





629



  TSDB_CHECK_CODE(code, lino, _end);




870,571,286✔





630












631



_end:




870,571,286✔





632



  if (code != TSDB_CODE_SUCCESS) {




1,455,595,832✔





633



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







634



  }










635



  return code;




1,455,353,969✔





636



}










637












638



void clearBrinBlockIter(SBrinRecordIter* pIter) {




592,342,505✔





639



  if (pIter != NULL) {




592,342,505✔





640



    tBrinBlockDestroy(&pIter->block);




592,525,257✔





641



  }










642



}




592,399,482✔





643












644



// initialize the file block access order










645



//  sort the file blocks according to the offset of each data block in the files










646



static void cleanupBlockOrderSupporter(SBlockOrderSupporter* pSup) {




23,473,152✔





647



  if (pSup == NULL) {




23,473,152✔





648



    return;




×







649



  }










650












651



  taosMemoryFreeClear(pSup->numOfBlocksPerTable);




23,473,152✔





652



  taosMemoryFreeClear(pSup->indexPerTable);




23,474,876✔





653












654



  if (pSup->pDataBlockInfo != NULL) {




23,471,800✔





655



    for (int32_t i = 0; i < pSup->numOfTables; ++i) {




70,896,514✔





656



      SBlockOrderWrapper* pBlockInfo = pSup->pDataBlockInfo[i];




47,422,838✔





657



      taosMemoryFreeClear(pBlockInfo);




47,423,354✔





658



    }










659












660



    taosMemoryFreeClear(pSup->pDataBlockInfo);




23,475,400✔





661



  }










662



}










663












664



static int32_t initBlockOrderSupporter(SBlockOrderSupporter* pSup, int32_t numOfTables) {




23,472,780✔





665



  int32_t code = TSDB_CODE_SUCCESS;




23,472,780✔





666



  int32_t lino = 0;




23,472,780✔





667












668



  TSDB_CHECK_NULL(pSup, code, lino, _end, TSDB_CODE_INVALID_PARA);




23,472,780✔





669



  TSDB_CHECK_CONDITION(numOfTables >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA);




23,472,780✔





670












671



  pSup->pDataBlockInfo = taosMemoryCalloc(numOfTables, POINTER_BYTES);




23,472,780✔





672



  TSDB_CHECK_NULL(pSup->pDataBlockInfo, code, lino, _end, terrno);




23,473,668✔





673



  pSup->indexPerTable = taosMemoryCalloc(numOfTables, sizeof(int32_t));




23,473,516✔





674



  TSDB_CHECK_NULL(pSup->indexPerTable, code, lino, _end, terrno);




23,477,640✔





675



  pSup->numOfBlocksPerTable = taosMemoryCalloc(numOfTables, sizeof(int32_t));




23,472,832✔





676



  TSDB_CHECK_NULL(pSup->numOfBlocksPerTable, code, lino, _end, terrno);




23,477,124✔





677



  pSup->numOfTables = 0;




23,473,880✔





678












679



_end:




23,473,364✔





680



  if (code != TSDB_CODE_SUCCESS) {




23,473,364✔





681



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







682



  }










683



  return code;




23,474,192✔





684



}










685












686



static int32_t fileDataBlockOrderCompar(const void* pLeft, const void* pRight, void* param) {




123,531,801✔





687



  int32_t                     leftIndex = 0;




123,531,801✔





688



  int32_t                     rightIndex = 0;




123,531,801✔





689



  int32_t                     leftTableBlockIndex = 0;




123,531,801✔





690



  int32_t                     rightTableBlockIndex = 0;




123,531,801✔





691



  const SBlockOrderSupporter* pSupporter = NULL;




123,531,801✔





692



  const SBlockOrderWrapper*   pLeftBlock = NULL;




123,531,801✔





693



  const SBlockOrderWrapper*   pRightBlock = NULL;




123,531,801✔





694












695



  leftIndex = *(const int32_t*)pLeft;




123,531,801✔





696



  rightIndex = *(const int32_t*)pRight;




123,509,418✔





697



  pSupporter = (const SBlockOrderSupporter*)param;




123,534,255✔





698












699



  leftTableBlockIndex = pSupporter->indexPerTable[leftIndex];




123,534,255✔





700



  rightTableBlockIndex = pSupporter->indexPerTable[rightIndex];




123,542,581✔





701












702



  if (leftTableBlockIndex >= pSupporter->numOfBlocksPerTable[leftIndex]) {




123,539,775✔





703



    /* left block is empty */










704



    return 1;




52,895,097✔





705



  } else if (rightTableBlockIndex >= pSupporter->numOfBlocksPerTable[rightIndex]) {




70,664,932✔





706



    /* right block is empty */










707



    return -1;




23,948,638✔





708



  }










709












710



  pLeftBlock = &pSupporter->pDataBlockInfo[leftIndex][leftTableBlockIndex];




46,716,392✔





711



  pRightBlock = &pSupporter->pDataBlockInfo[rightIndex][rightTableBlockIndex];




46,715,652✔





712












713



  return pLeftBlock->offset > pRightBlock->offset ? 1 : -1;




46,717,380✔





714



}










715












716



int32_t recordToBlockInfo(SFileDataBlockInfo* pBlockInfo, SBrinRecord* record) {




307,797,737✔





717



  int32_t        code = TSDB_CODE_SUCCESS;




307,797,737✔





718



  int32_t        lino = 0;




307,797,737✔





719



  const SRowKey* pFirstKey = NULL;




307,797,737✔





720



  const SRowKey* pLastKey = NULL;




307,797,737✔





721












722



  TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




307,797,737✔





723



  TSDB_CHECK_NULL(record, code, lino, _end, TSDB_CODE_INVALID_PARA);




307,797,737✔





724












725



  pBlockInfo->uid = record->uid;




307,797,737✔





726



  pBlockInfo->firstKey = record->firstKey.key.ts;




307,892,006✔





727



  pBlockInfo->lastKey = record->lastKey.key.ts;




307,918,793✔





728



  pBlockInfo->minVer = record->minVer;




307,921,494✔





729



  pBlockInfo->maxVer = record->maxVer;




307,937,536✔





730



  pBlockInfo->blockOffset = record->blockOffset;




307,921,764✔





731



  pBlockInfo->smaOffset = record->smaOffset;




307,933,595✔





732



  pBlockInfo->blockSize = record->blockSize;




307,923,978✔





733



  pBlockInfo->blockKeySize = record->blockKeySize;




307,933,292✔





734



  pBlockInfo->smaSize = record->smaSize;




307,922,323✔





735



  pBlockInfo->numRow = record->numRow;




307,932,877✔





736



  pBlockInfo->count = record->count;




307,919,550✔





737












738



  pFirstKey = &record->firstKey.key;




307,936,592✔





739



  pLastKey = &record->lastKey.key;




307,932,028✔





740



  TSDB_CHECK_CONDITION((pFirstKey->numOfPKs == pLastKey->numOfPKs), code, lino, _end, TSDB_CODE_INVALID_PARA);




307,915,328✔





741



  if (pFirstKey->numOfPKs > 0) {




307,922,607✔





742



    if (IS_NUMERIC_TYPE(pFirstKey->pks[0].type)) {




69,082✔





743



      pBlockInfo->firstPk.val = VALUE_GET_TRIVIAL_DATUM(pFirstKey->pks);




×







744



      pBlockInfo->lastPk.val = VALUE_GET_TRIVIAL_DATUM(pLastKey->pks);




×







745



    } else {










746



      int32_t keyLen = pFirstKey->pks[0].nData;




69,082✔





747



      char*   p = taosMemoryMalloc(keyLen + VARSTR_HEADER_SIZE);




69,082✔





748



      TSDB_CHECK_NULL(p, code, lino, _end, terrno);




69,082✔





749



      memcpy(varDataVal(p), pFirstKey->pks[0].pData, keyLen);




69,082✔





750



      varDataSetLen(p, keyLen);




69,082✔





751



      pBlockInfo->firstPk.pData = (uint8_t*)p;




69,082✔





752












753



      keyLen = pLastKey->pks[0].nData;




69,082✔





754



      p = taosMemoryMalloc(keyLen + VARSTR_HEADER_SIZE);




69,082✔





755



      TSDB_CHECK_NULL(p, code, lino, _end, terrno);




69,082✔





756



      memcpy(varDataVal(p), pLastKey->pks[0].pData, keyLen);




69,082✔





757



      varDataSetLen(p, keyLen);




69,082✔





758



      pBlockInfo->lastPk.pData = (uint8_t*)p;




69,082✔





759



    }










760



  }










761












762



_end:




307,780,047✔





763



  if (code != TSDB_CODE_SUCCESS) {




307,849,129✔





764



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







765



  }










766



  return code;




307,860,484✔





767



}










768












769



static void freePkItem(void* pItem) {




69,082✔





770



  SFileDataBlockInfo* p = pItem;




69,082✔





771



  if (p != NULL) {




69,082✔





772



    taosMemoryFreeClear(p->firstPk.pData);




69,082✔





773



    taosMemoryFreeClear(p->lastPk.pData);




69,082✔





774



  }










775



}




69,082✔





776












777



void clearDataBlockIterator(SDataBlockIter* pIter, bool needFree) {




635,919,357✔





778



  if (pIter == NULL) {




635,919,357✔





779



    return;




×







780



  }










781












782



  pIter->index = -1;




635,919,357✔





783



  pIter->numOfBlocks = 0;




636,258,708✔





784












785



  if (needFree) {




636,027,055✔





786



    taosArrayClearEx(pIter->blockList, freePkItem);




92,186,452✔





787



  } else {










788



    taosArrayClear(pIter->blockList);




543,840,603✔





789



  }










790



}










791












792



void cleanupDataBlockIterator(SDataBlockIter* pIter, bool needFree) {




322,262,453✔





793



  if (pIter == NULL) {




322,262,453✔





794



    return;




×







795



  }










796












797



  pIter->index = -1;




322,262,453✔





798



  pIter->numOfBlocks = 0;




322,298,647✔





799



  if (needFree) {




322,260,082✔





800



    taosArrayDestroyEx(pIter->blockList, freePkItem);




5,561,260✔





801



  } else {










802



    taosArrayDestroy(pIter->blockList);




316,698,822✔





803



  }










804



  pIter->blockList = NULL;




322,175,169✔





805



}










806












807



int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList) {




23,473,456✔





808



  int32_t                   code = TSDB_CODE_SUCCESS;




23,473,456✔





809



  int32_t                   lino = 0;




23,473,456✔





810



  bool                      asc = false;




23,473,456✔





811



  int32_t                   numOfTables = 0;




23,473,456✔





812



  int64_t                   st = 0;




23,473,456✔





813



  int64_t                   et = 0;




23,473,456✔





814



  int32_t                   cnt = 0;




23,473,456✔





815



  SBlockOrderSupporter      sup = {0};




23,473,456✔





816



  SMultiwayMergeTreeInfo*   pTree = NULL;




23,474,876✔





817



  STableBlockScanInfo*      pTableScanInfo = NULL;




23,473,456✔





818



  const SFileDataBlockInfo* pBlockInfo = NULL;




23,473,456✔





819



  const void*               px = NULL;




23,473,456✔





820












821



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




23,473,456✔





822



  TSDB_CHECK_NULL(pBlockIter, code, lino, _end, TSDB_CODE_INVALID_PARA);




23,473,456✔





823



  TSDB_CHECK_CONDITION(numOfBlocks >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA);




23,473,456✔





824












825



  asc = ASCENDING_TRAVERSE(pReader->info.order);




23,473,456✔





826



  clearDataBlockIterator(pBlockIter, shouldFreePkBuf(&pReader->suppInfo));




23,472,096✔





827



  pBlockIter->numOfBlocks = numOfBlocks;




23,475,916✔





828












829



  // access data blocks according to the offset of each block in asc/desc order.










830



  numOfTables = taosArrayGetSize(pTableList);




23,474,352✔





831












832



  st = taosGetTimestampUs();




23,474,200✔





833



  code = initBlockOrderSupporter(&sup, numOfTables);




23,474,200✔





834



  TSDB_CHECK_CODE(code, lino, _end);




23,474,564✔





835












836



  for (int32_t i = 0; i < numOfTables; ++i) {




70,896,134✔





837



    pTableScanInfo = taosArrayGetP(pTableList, i);




47,421,942✔





838












839



    size_t              num = taosArrayGetSize(pTableScanInfo->pBlockList);




47,425,914✔





840



    SBlockOrderWrapper* buf = taosMemoryMalloc(sizeof(SBlockOrderWrapper) * num);




47,423,294✔





841



    TSDB_CHECK_NULL(buf, code, lino, _end, terrno);




47,416,618✔





842



    sup.numOfBlocksPerTable[sup.numOfTables] = num;




47,416,618✔





843



    sup.pDataBlockInfo[sup.numOfTables] = buf;




47,421,266✔





844



    sup.numOfTables++;




47,423,142✔





845












846



    for (int32_t k = 0; k < num; ++k) {




355,113,242✔





847



      pBlockInfo = taosArrayGet(pTableScanInfo->pBlockList, k);




307,691,672✔





848



      TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




307,636,680✔





849












850



      sup.pDataBlockInfo[i][k] =




615,333,632✔





851



          (SBlockOrderWrapper){.uid = pTableScanInfo->uid, .offset = pBlockInfo->blockOffset, .pInfo = pTableScanInfo};




307,672,972✔





852



      cnt++;




307,690,100✔





853



    }










854



  }










855












856



  TSDB_CHECK_CONDITION(!(numOfBlocks != cnt && sup.numOfTables != numOfTables), code, lino, _end,




23,474,192✔





857



                       TSDB_CODE_INVALID_PARA);










858












859



  // since there is only one table qualified, blocks are not sorted










860



  if (sup.numOfTables == 1) {




23,474,192✔





861



    pTableScanInfo = taosArrayGetP(pTableList, 0);




15,135,779✔





862



    for (int32_t i = 0; i < numOfBlocks; ++i) {




272,082,286✔





863



      STableDataBlockIdx tableDataBlockIdx = {.globalIndex = i};




256,939,139✔





864



      px = taosArrayPush(pTableScanInfo->pBlockIdxList, &tableDataBlockIdx);




256,948,443✔





865



      TSDB_CHECK_NULL(px, code, lino, _end, terrno);




256,937,643✔





866



    }










867












868



    px = taosArrayAddAll(pBlockIter->blockList, pTableScanInfo->pBlockList);




15,143,147✔





869



    TSDB_CHECK_NULL(px, code, lino, _end, terrno);




15,138,703✔





870












871



    taosArrayDestroy(pTableScanInfo->pBlockList);




15,138,703✔





872



    pTableScanInfo->pBlockList = NULL;




15,135,323✔





873












874



    et = taosGetTimestampUs();




15,136,827✔





875



    tsdbDebug("%p create blocks info struct completed for one table, %d blocks not sorted, elapsed time:%.2f ms %s",




15,136,827✔





876



              pReader, numOfBlocks, (et - st) / 1000.0, pReader->idStr);










877












878



    pBlockIter->index = asc ? 0 : (numOfBlocks - 1);




15,135,779✔





879



    goto _end;




15,135,999✔





880



  }










881












882



  tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables %s", pReader, cnt, sup.numOfTables,




8,338,413✔





883



            pReader->idStr);










884












885



  code = tMergeTreeCreate(&pTree, sup.numOfTables, &sup, fileDataBlockOrderCompar);




8,338,413✔





886



  TSDB_CHECK_CODE(code, lino, _end);




8,339,773✔





887












888



  for (int32_t i = 0; i < cnt; ++i) {




59,376,366✔





889



    int32_t pos = tMergeTreeGetChosenIndex(pTree);




51,036,396✔





890



    int32_t index = sup.indexPerTable[pos]++;




51,036,207✔





891



    pTableScanInfo = sup.pDataBlockInfo[pos][index].pInfo;




51,036,201✔





892












893



    pBlockInfo = taosArrayGet(pTableScanInfo->pBlockList, index);




51,035,851✔





894



    TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, terrno);




51,038,389✔





895












896



    px = taosArrayPush(pBlockIter->blockList, pBlockInfo);




51,038,389✔





897



    TSDB_CHECK_NULL(px, code, lino, _end, terrno);




51,038,739✔





898












899



    STableDataBlockIdx tableDataBlockIdx = {.globalIndex = i};




51,038,739✔





900



    px = taosArrayPush(pTableScanInfo->pBlockIdxList, &tableDataBlockIdx);




51,038,709✔





901



    TSDB_CHECK_NULL(px, code, lino, _end, terrno);




51,041,200✔





902












903



    // set data block index overflow, in order to disable the offset comparator










904



    if (sup.indexPerTable[pos] >= sup.numOfBlocksPerTable[pos]) {




51,041,200✔





905



      sup.indexPerTable[pos] = sup.numOfBlocksPerTable[pos] + 1;




32,287,887✔





906



    }










907












908



    code = tMergeTreeAdjust(pTree, tMergeTreeGetAdjustIndex(pTree));




51,042,251✔





909



    TSDB_CHECK_CODE(code, lino, _end);




51,035,849✔





910



  }










911












912



  for (int32_t i = 0; i < numOfTables; ++i) {




40,627,660✔





913



    pTableScanInfo = taosArrayGetP(pTableList, i);




32,287,887✔





914



    taosArrayDestroy(pTableScanInfo->pBlockList);




32,288,411✔





915



    pTableScanInfo->pBlockList = NULL;




32,288,411✔





916



  }










917












918



  et = taosGetTimestampUs();




8,339,773✔





919



  tsdbDebug("%p %d data blocks access order completed, elapsed time:%.2f ms %s", pReader, numOfBlocks,




8,339,773✔





920



            (et - st) / 1000.0, pReader->idStr);










921












922



  pBlockIter->index = asc ? 0 : (numOfBlocks - 1);




8,339,773✔





923












924



_end:




23,469,993✔





925



  if (code != TSDB_CODE_SUCCESS) {




23,474,504✔





926



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







927



  }










928



  cleanupBlockOrderSupporter(&sup);




23,474,504✔





929



  if (pTree != NULL) {




23,472,992✔





930



    tMergeTreeDestroy(&pTree);




8,339,773✔





931



  }










932



  return code;




23,472,992✔





933



}










934












935



bool blockIteratorNext(SDataBlockIter* pBlockIter) {




129,634,858✔





936



  bool asc = false;




129,634,858✔





937












938



  if (pBlockIter == NULL) {




129,634,858✔





939



    return false;




×







940



  }










941












942



  asc = ASCENDING_TRAVERSE(pBlockIter->order);




129,634,858✔





943



  if ((pBlockIter->index >= pBlockIter->numOfBlocks - 1 && asc) || (pBlockIter->index <= 0 && (!asc))) {




129,669,113✔





944



    return false;




19,760,349✔





945



  }










946



  pBlockIter->index += asc ? 1 : -1;




109,911,412✔





947



  return true;




109,906,560✔





948



}










949












950



typedef enum {










951



  BLK_CHECK_CONTINUE = 0x1,










952



  BLK_CHECK_QUIT = 0x2,










953



} ETombBlkCheckEnum;










954












955



static int32_t loadNextStatisticsBlock(SSttFileReader* pSttFileReader, STbStatisBlock* pStatisBlock,










956



                                       const TStatisBlkArray* pStatisBlkArray, int32_t numOfRows, int32_t* i,










957



                                       int32_t* j);










958



static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_t numOfTables, int32_t* j,




6,837,213✔





959



                                ETombBlkCheckEnum* pRet) {










960



  int32_t              code = TSDB_CODE_SUCCESS;




6,837,213✔





961



  int32_t              lino = 0;




6,837,213✔





962



  STombRecord          record = {0};




6,837,213✔





963



  uint64_t             uid = 0;




6,837,213✔





964



  STableBlockScanInfo* pScanInfo = NULL;




6,837,213✔





965












966



  TSDB_CHECK_NULL(pBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);




6,837,213✔





967



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




6,837,213✔





968



  TSDB_CHECK_NULL(pRet, code, lino, _end, TSDB_CODE_INVALID_PARA);




6,837,213✔





969












970



  *pRet = BLK_CHECK_QUIT;




6,837,213✔





971



  uid = pReader->status.uidList.tableUidList[*j];




6,837,213✔





972



  code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr);




6,837,213✔





973



  TSDB_CHECK_CODE(code, lino, _end);




6,837,213✔





974












975



  if (pScanInfo->pFileDelData == NULL) {




6,837,213✔





976



    pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData));




3,284,255✔





977



    TSDB_CHECK_NULL(pScanInfo->pFileDelData, code, lino, _end, terrno);




3,284,255✔





978



  }










979












980



  for (int32_t k = 0; k < pBlock->numOfRecords; ++k) {




636,272,904✔





981



    code = tTombBlockGet(pBlock, k, &record);




629,538,219✔





982



    TSDB_CHECK_CODE(code, lino, _end);




629,538,219✔





983












984



    if (record.suid < pReader->info.suid) {




629,538,219✔





985



      continue;




564,108,400✔





986



    }










987












988



    if (record.suid > pReader->info.suid) {




65,429,819✔





989



      *pRet = BLK_CHECK_QUIT;




×







990



      goto _end;




×







991



    }










992












993



    if (uid < record.uid) {




65,429,819✔





994



      while ((*j) < numOfTables && pReader->status.uidList.tableUidList[*j] < record.uid) {




6,640,256✔





995



        (*j) += 1;




3,339,768✔





996



      }










997












998



      if ((*j) >= numOfTables) {




3,300,488✔





999



        *pRet = BLK_CHECK_QUIT;




102,528✔





1000



        goto _end;




102,528✔





1001



      }










1002












1003



      uid = pReader->status.uidList.tableUidList[*j];




3,197,960✔





1004



      code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr);




3,197,960✔





1005



      TSDB_CHECK_CODE(code, lino, _end);




3,197,960✔





1006












1007



      if (pScanInfo->pFileDelData == NULL) {




3,197,960✔





1008



        pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData));




1,757,142✔





1009



        TSDB_CHECK_NULL(pScanInfo->pFileDelData, code, lino, _end, terrno);




1,757,142✔





1010



      }










1011



    }










1012












1013



    if (record.uid < uid) {




65,327,291✔





1014



      continue;




1,314,708✔





1015



    }










1016












1017



    TSDB_CHECK_CONDITION((record.suid == pReader->info.suid) && (uid == record.uid), code, lino, _end,




64,012,583✔





1018



                         TSDB_CODE_INTERNAL_ERROR);










1019












1020



    if (record.version <= pReader->info.verRange.maxVer) {




64,012,583✔





1021



      SDelData    delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey};




64,012,583✔





1022



      const void* px = taosArrayPush(pScanInfo->pFileDelData, &delData);




64,012,583✔





1023



      TSDB_CHECK_NULL(px, code, lino, _end, terrno);




64,012,583✔





1024



    }










1025



  }










1026












1027



  *pRet = BLK_CHECK_CONTINUE;




6,734,685✔





1028












1029



_end:




6,837,213✔





1030



  if (code != TSDB_CODE_SUCCESS) {




6,837,213✔





1031



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1032



  }










1033



  return code;




6,837,213✔





1034



}










1035












1036



// load tomb data API










1037



static int32_t doLoadTombDataFromTombBlk(const TTombBlkArray* pTombBlkArray, STsdbReader* pReader, void* pFileReader,




577,583,089✔





1038



                                         bool isFile) {










1039



  int32_t              code = TSDB_CODE_SUCCESS;




577,583,089✔





1040



  int32_t              lino = 0;




577,583,089✔





1041



  const STableUidList* pList = NULL;




577,583,089✔





1042



  int32_t              numOfTables = 0;




577,583,089✔





1043












1044



  TSDB_CHECK_NULL(pTombBlkArray, code, lino, _end, TSDB_CODE_INVALID_PARA);




577,583,089✔





1045



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




577,583,089✔





1046












1047



  pList = &pReader->status.uidList;




577,583,089✔





1048



  numOfTables = tSimpleHashGetSize(pReader->status.pTableMap);




577,667,520✔





1049












1050



  int32_t i = 0, j = 0;




577,913,727✔





1051



  while (i < pTombBlkArray->size && j < numOfTables) {




593,310,423✔





1052



    STombBlk* pTombBlk = &pTombBlkArray->data[i];




16,681,152✔





1053



    if (pTombBlk->maxTbid.suid < pReader->info.suid) {




16,681,152✔





1054



      i += 1;




5,332,115✔





1055



      continue;




5,332,115✔





1056



    }










1057












1058



    if (pTombBlk->minTbid.suid > pReader->info.suid) {




11,349,037✔





1059



      break;




1,021,646✔





1060



    }










1061












1062



    TSDB_CHECK_CONDITION(




10,327,391✔





1063



        (pTombBlk->minTbid.suid <= pReader->info.suid) && (pTombBlk->maxTbid.suid >= pReader->info.suid), code, lino,










1064



        _end, TSDB_CODE_INTERNAL_ERROR);










1065












1066



    if (pTombBlk->maxTbid.suid == pReader->info.suid && pTombBlk->maxTbid.uid < pList->tableUidList[0]) {




10,327,391✔





1067



      i += 1;




3,355,930✔





1068



      continue;




3,355,930✔





1069



    }










1070












1071



    if (pTombBlk->minTbid.suid == pReader->info.suid && pTombBlk->minTbid.uid > pList->tableUidList[numOfTables - 1]) {




6,971,461✔





1072



      break;




134,248✔





1073



    }










1074












1075



    STombBlock block = {0};




6,837,213✔





1076



    code = isFile ? tsdbDataFileReadTombBlock(pFileReader, &pTombBlkArray->data[i], &block)




1,912,264✔





1077



                  : tsdbSttFileReadTombBlock(pFileReader, &pTombBlkArray->data[i], &block);




6,837,213✔





1078



    TSDB_CHECK_CODE(code, lino, _end);




6,837,213✔





1079












1080



    ETombBlkCheckEnum ret = 0;




6,837,213✔





1081



    code = doCheckTombBlock(&block, pReader, numOfTables, &j, &ret);




6,837,213✔





1082












1083



    tTombBlockDestroy(&block);




6,837,213✔





1084



    TSDB_CHECK_CODE(code, lino, _end);




6,837,213✔





1085



    if (ret == BLK_CHECK_QUIT) {




6,837,213✔





1086



      break;




102,528✔





1087



    }










1088












1089



    i += 1;




6,734,685✔





1090



  }










1091












1092



_end:




577,844,508✔





1093



  if (code != TSDB_CODE_SUCCESS) {




577,871,098✔





1094



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1095



  }










1096



  return code;




577,871,068✔





1097



}










1098












1099



int32_t loadDataFileTombDataForAll(STsdbReader* pReader) {




667,965,199✔





1100



  int32_t              code = TSDB_CODE_SUCCESS;




667,965,199✔





1101



  int32_t              lino = 0;




667,965,199✔





1102



  const TTombBlkArray* pBlkArray = NULL;




667,965,199✔





1103












1104



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




668,061,875✔





1105












1106



  if ((pReader->status.pCurrentFileset == NULL) || (pReader->status.pCurrentFileset->farr[TSDB_FTYPE_TOMB] == NULL)) {




668,061,875✔





1107



    return TSDB_CODE_SUCCESS;




662,937,455✔





1108



  }










1109












1110



  code = tsdbDataFileReadTombBlk(pReader->pFileReader, &pBlkArray);




5,307,240✔





1111



  TSDB_CHECK_CODE(code, lino, _end);




5,307,240✔





1112












1113



  code = doLoadTombDataFromTombBlk(pBlkArray, pReader, pReader->pFileReader, true);




5,307,240✔





1114



  TSDB_CHECK_CODE(code, lino, _end);




5,307,240✔





1115












1116



_end:




5,307,240✔





1117



  if (code != TSDB_CODE_SUCCESS) {




5,307,240✔





1118



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1119



  }










1120



  return code;




5,307,240✔





1121



}










1122












1123



int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pLoadInfo) {




572,245,290✔





1124



  int32_t              code = TSDB_CODE_SUCCESS;




572,245,290✔





1125



  int32_t              lino = 0;




572,245,290✔





1126



  const TTombBlkArray* pBlkArray = NULL;




572,245,290✔





1127












1128



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




572,391,420✔





1129












1130



  code = tsdbSttFileReadTombBlk(pSttFileReader, &pBlkArray);




572,391,420✔





1131



  TSDB_CHECK_CODE(code, lino, _end);




572,458,579✔





1132












1133



  code = doLoadTombDataFromTombBlk(pBlkArray, pReader, pSttFileReader, false);




572,458,579✔





1134



  TSDB_CHECK_CODE(code, lino, _end);




572,476,068✔





1135












1136



_end:




572,476,068✔





1137



  if (code != TSDB_CODE_SUCCESS) {




572,527,144✔





1138



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1139



  }










1140



  return code;




572,527,144✔





1141



}










1142












1143



int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver) {




761,014,830✔





1144



  int32_t         code = TSDB_CODE_SUCCESS;




761,014,830✔





1145



  int32_t         lino = 0;




761,014,830✔





1146



  SArray*         pMemDelData = NULL;




761,014,830✔





1147



  const SDelData* p = NULL;




761,014,830✔





1148



  const void*     px = NULL;




761,014,830✔





1149












1150



  TSDB_CHECK_NULL(ppMemDelData, code, lino, _end, TSDB_CODE_INVALID_PARA);




761,014,830✔





1151












1152



  if (*ppMemDelData == NULL) {




761,014,830✔





1153



    *ppMemDelData = taosArrayInit(4, sizeof(SDelData));




755,512,818✔





1154



    TSDB_CHECK_NULL(*ppMemDelData, code, lino, _end, terrno);




755,827,742✔





1155



  }










1156












1157



  pMemDelData = *ppMemDelData;




761,558,627✔





1158












1159



  if (pMemTbData != NULL) {




761,061,748✔





1160



    taosRLockLatch(&pMemTbData->lock);




463,310,837✔





1161



    p = pMemTbData->pHead;




463,434,602✔





1162



    while (p) {




464,987,369✔





1163



      if (p->version <= ver) {




1,718,870✔





1164



        px = taosArrayPush(pMemDelData, p);




1,663,964✔





1165



        if (px == NULL) {




1,663,964✔





1166



          taosRUnLockLatch(&pMemTbData->lock);




×







1167



          TSDB_CHECK_NULL(px, code, lino, _end, terrno);




×







1168



        }










1169



      }










1170



      p = p->pNext;




1,719,582✔





1171



    }










1172



    taosRUnLockLatch(&pMemTbData->lock);




463,268,499✔





1173



  }










1174












1175



  if (piMemTbData != NULL) {




761,176,146✔





1176



    p = piMemTbData->pHead;




13,831,853✔





1177



    while (p) {




14,958,083✔





1178



      if (p->version <= ver) {




1,126,160✔





1179



        px = taosArrayPush(pMemDelData, p);




1,126,160✔





1180



        TSDB_CHECK_NULL(px, code, lino, _end, terrno);




1,126,160✔





1181



      }










1182



      p = p->pNext;




1,126,160✔





1183



    }










1184



  }










1185












1186



_end:




761,176,216✔





1187



  if (code != TSDB_CODE_SUCCESS) {




761,176,216✔





1188



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1189



  }










1190



  return code;




760,981,217✔





1191



}










1192












1193



int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo,




3,912✔





1194



                               TStatisBlkArray* pStatisBlkArray, uint64_t suid, const uint64_t* pUidList,










1195



                               int32_t numOfTables, int32_t* pNumOfRows) {










1196



  int32_t           code = TSDB_CODE_SUCCESS;




3,912✔





1197



  int32_t           lino = 0;




3,912✔





1198



  int32_t           num = 0;




3,912✔





1199



  int64_t           st = 0;




3,912✔





1200



  const SStatisBlk* p = NULL;




3,912✔





1201



  STbStatisBlock*   pStatisBlock = NULL;




3,912✔





1202












1203



  TSDB_CHECK_NULL(pSttFileReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




3,912✔





1204



  TSDB_CHECK_NULL(pBlockLoadInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




3,912✔





1205



  TSDB_CHECK_NULL(pStatisBlkArray, code, lino, _end, TSDB_CODE_INVALID_PARA);




3,912✔





1206



  TSDB_CHECK_NULL(pNumOfRows, code, lino, _end, TSDB_CODE_INVALID_PARA);




3,912✔





1207












1208



  *pNumOfRows = 0;




3,912✔





1209












1210



  if (TARRAY2_SIZE(pStatisBlkArray) <= 0) {




3,912✔





1211



    return code;




×







1212



  }










1213












1214



  int32_t i = 0;




3,912✔





1215



  while ((i < TARRAY2_SIZE(pStatisBlkArray)) && (pStatisBlkArray->data[i].maxTbid.suid < suid)) {




3,912✔





1216



    ++i;




×







1217



  }










1218












1219



  if (i >= TARRAY2_SIZE(pStatisBlkArray)) {




3,912✔





1220



    goto _end;




×







1221



  }










1222












1223



  p = &pStatisBlkArray->data[i];




3,912✔





1224



  pStatisBlock = taosMemoryCalloc(1, sizeof(STbStatisBlock));




3,912✔





1225



  TSDB_CHECK_NULL(pStatisBlock, code, lino, _end, terrno);




3,912✔





1226












1227



  code = tStatisBlockInit(pStatisBlock);




3,912✔





1228



  TSDB_CHECK_CODE(code, lino, _end);




3,912✔





1229












1230



  st = taosGetTimestampUs();




3,912✔





1231



  code = tsdbSttFileReadStatisBlock(pSttFileReader, p, pStatisBlock);




3,912✔





1232



  TSDB_CHECK_CODE(code, lino, _end);




3,912✔





1233












1234



  double el = (taosGetTimestampUs() - st) / 1000.0;




3,912✔





1235



  pBlockLoadInfo->cost.loadStatisBlocks += 1;




3,912✔





1236



  pBlockLoadInfo->cost.statisElapsedTime += el;




3,912✔





1237












1238



  int32_t index = 0;




3,912✔





1239



  while (index < pStatisBlock->numOfRecords && ((int64_t*)pStatisBlock->suids.data)[index] < suid) {




3,912✔





1240



    ++index;




×







1241



  }










1242












1243



  if (index >= pStatisBlock->numOfRecords) {




3,912✔





1244



    *pNumOfRows = num;




×







1245



    goto _end;




×







1246



  }










1247












1248



  int32_t j = index;




3,912✔





1249



  int32_t uidIndex = 0;




3,912✔





1250



  while (i < TARRAY2_SIZE(pStatisBlkArray) && uidIndex < numOfTables) {




7,259✔





1251



    p = &pStatisBlkArray->data[i];




4,017✔





1252



    if (p->minTbid.suid > suid) {




4,017✔





1253



      *pNumOfRows = num;




670✔





1254



      goto _end;




670✔





1255



    }










1256












1257



    uint64_t uid = pUidList[uidIndex];




3,347✔





1258












1259



    if (((int64_t*)pStatisBlock->uids.data)[j] == uid) {




3,347✔





1260



      num += ((int64_t*)pStatisBlock->counts.data)[j];




3,347✔





1261



      uidIndex += 1;




3,347✔





1262



      j += 1;




3,347✔





1263



      code = loadNextStatisticsBlock(pSttFileReader, pStatisBlock, pStatisBlkArray, pStatisBlock->numOfRecords, &i, &j);




3,347✔





1264



      TSDB_CHECK_CODE(code, lino, _end);




3,347✔





1265



    } else if (((int64_t*)pStatisBlock->uids.data)[j] < uid) {




×







1266



      j += 1;




×







1267



      code = loadNextStatisticsBlock(pSttFileReader, pStatisBlock, pStatisBlkArray, pStatisBlock->numOfRecords, &i, &j);




×







1268



      TSDB_CHECK_CODE(code, lino, _end);




×







1269



    } else {










1270



      uidIndex += 1;




×







1271



    }










1272



  }










1273












1274



  *pNumOfRows = num;




3,242✔





1275












1276



_end:




3,912✔





1277



  if (code != TSDB_CODE_SUCCESS) {




3,912✔





1278



    tsdbError("%s with %p failed at line %d since %s", __func__, pSttFileReader, lino, tstrerror(code));




×







1279



  }










1280



  if (pStatisBlock) {




3,912✔





1281



    tStatisBlockDestroy(pStatisBlock);




3,912✔





1282



    taosMemoryFreeClear(pStatisBlock);




3,912✔





1283



  }










1284



  return code;




3,912✔





1285



}










1286












1287



// load next stt statistics block










1288



static int32_t loadNextStatisticsBlock(SSttFileReader* pSttFileReader, STbStatisBlock* pStatisBlock,




3,347✔





1289



                                       const TStatisBlkArray* pStatisBlkArray, int32_t numOfRows, int32_t* i,










1290



                                       int32_t* j) {










1291



  int32_t code = TSDB_CODE_SUCCESS;




3,347✔





1292



  int32_t lino = 0;




3,347✔





1293












1294



  if ((*j) >= numOfRows) {




3,347✔





1295



    (*i) += 1;




3,242✔





1296



    (*j) = 0;




3,242✔





1297



    if ((*i) < TARRAY2_SIZE(pStatisBlkArray)) {




3,242✔





1298



      code = tsdbSttFileReadStatisBlock(pSttFileReader, &pStatisBlkArray->data[(*i)], pStatisBlock);




×







1299



      if (code != 0) {




×







1300



        tsdbError("%p failed to read statisBlock, code:%s", pSttFileReader, tstrerror(code));




×







1301



        TSDB_CHECK_CODE(code, lino, _end);




×







1302



      }










1303



    }










1304



  }










1305












1306



_end:




3,347✔





1307



  if (code != TSDB_CODE_SUCCESS) {




3,347✔





1308



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1309



  }










1310



  return code;




3,347✔





1311



}










1312












1313



int32_t doAdjustValidDataIters(SArray* pLDIterList, int32_t numOfFileObj) {




1,086,957,081✔





1314



  int32_t     code = TSDB_CODE_SUCCESS;




1,086,957,081✔





1315



  int32_t     lino = 0;




1,086,957,081✔





1316



  int32_t     size = 0;




1,086,957,081✔





1317



  int32_t     inc = 0;




1,086,957,081✔





1318



  SLDataIter* pIter = NULL;




1,086,957,081✔





1319












1320



  size = taosArrayGetSize(pLDIterList);




1,087,067,734✔





1321












1322



  if (size < numOfFileObj) {




1,087,038,492✔





1323



    inc = numOfFileObj - size;




573,589,643✔





1324



    for (int32_t k = 0; k < inc; ++k) {




1,147,496,283✔





1325



      pIter = taosMemoryCalloc(1, sizeof(SLDataIter));




573,763,771✔





1326



      TSDB_CHECK_NULL(pIter, code, lino, _end, terrno);




573,140,604✔





1327



      void* px = taosArrayPush(pLDIterList, &pIter);




573,906,640✔





1328



      if (px == NULL) {




573,906,640✔





1329



        TSDB_CHECK_NULL(px, code, lino, _end, terrno);




×







1330



        taosMemoryFreeClear(pIter);




×







1331



      }










1332



    }










1333



  } else if (size > numOfFileObj) {  // remove unused LDataIter




513,448,849✔





1334



    inc = size - numOfFileObj;




×







1335












1336



    for (int32_t i = 0; i < inc; ++i) {




×







1337



      pIter = taosArrayPop(pLDIterList);




×







1338



      destroyLDataIter(pIter);




×







1339



    }










1340



  }










1341












1342



_end:




1,087,179,861✔





1343



  if (code != TSDB_CODE_SUCCESS) {




1,086,859,617✔





1344



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1345



  }










1346



  return code;




1,086,859,617✔





1347



}










1348












1349



int32_t adjustSttDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet) {




1,051,408,623✔





1350



  int32_t  code = TSDB_CODE_SUCCESS;




1,051,408,623✔





1351



  int32_t  lino = 0;




1,051,408,623✔





1352



  int32_t  numOfLevels = 0;




1,051,408,623✔





1353



  SSttLvl* pSttLevel = NULL;




1,051,408,623✔





1354



  SArray*  pList = NULL;




1,051,408,623✔





1355












1356



  TSDB_CHECK_NULL(pFileSet, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,051,607,724✔





1357












1358



  numOfLevels = pFileSet->lvlArr->size;




1,051,607,724✔





1359












1360



  // add the list/iter placeholder










1361



  while (taosArrayGetSize(pSttFileBlockIterArray) < numOfLevels) {




1,625,166,725✔





1362



    pList = taosArrayInit(4, POINTER_BYTES);




573,286,955✔





1363



    TSDB_CHECK_NULL(pList, code, lino, _end, terrno);




573,487,375✔





1364



    void* px = taosArrayPush(pSttFileBlockIterArray, &pList);




573,575,016✔





1365



    if (px == NULL) {




573,575,016✔





1366



      taosArrayDestroy(pList);




×







1367



      TSDB_CHECK_NULL(px, code, lino, _end, terrno);




×







1368



    }










1369



  }










1370












1371



  for (int32_t j = 0; j < numOfLevels; ++j) {




2,138,744,478✔





1372



    pSttLevel = pFileSet->lvlArr->data[j];




1,086,683,974✔





1373



    pList = taosArrayGetP(pSttFileBlockIterArray, j);




1,086,970,752✔





1374



    code = doAdjustValidDataIters(pList, TARRAY2_SIZE(pSttLevel->fobjArr));




1,086,984,890✔





1375



    TSDB_CHECK_CODE(code, lino, _end);




1,086,826,896✔





1376



  }










1377












1378



_end:




1,052,060,504✔





1379



  if (code != TSDB_CODE_SUCCESS) {




1,051,736,706✔





1380



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1381



  }










1382



  return code;




1,051,736,706✔





1383



}










1384












1385



int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArray, STsdb* pTsdb, SMergeTreeConf* pConf,




30,317✔





1386



                              const char* pstr) {










1387



  int32_t code = TSDB_CODE_SUCCESS;




30,317✔





1388



  int32_t lino = 0;




30,317✔





1389



  int32_t numOfRows = 0;




30,317✔





1390



  int32_t numOfLevels = 0;




30,317✔





1391












1392



  TSDB_CHECK_NULL(pFileSet, code, lino, _end, TSDB_CODE_INVALID_PARA);




30,317✔





1393












1394



  // no data exists, go to end










1395



  numOfLevels = pFileSet->lvlArr->size;




30,317✔





1396



  if (numOfLevels == 0) {




30,317✔





1397



    goto _end;




26,405✔





1398



  }










1399












1400



  // add the list/iter placeholder










1401



  code = adjustSttDataIters(pSttFileBlockIterArray, pFileSet);




3,912✔





1402



  TSDB_CHECK_CODE(code, lino, _end);




3,912✔





1403












1404



  for (int32_t j = 0; j < numOfLevels; ++j) {




7,824✔





1405



    SSttLvl* pSttLevel = pFileSet->lvlArr->data[j];




3,912✔





1406



    SArray*  pList = taosArrayGetP(pSttFileBlockIterArray, j);




3,912✔





1407












1408



    for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) {  // open all last file




7,824✔





1409



      SLDataIter* pIter = taosArrayGetP(pList, i);




3,912✔





1410












1411



      // open stt file reader if not opened yet










1412



      // if failed to open this stt file, ignore the error and try next one










1413



      if (pIter->pReader == NULL) {




3,912✔





1414



        SSttFileReaderConfig conf = {.tsdb = pTsdb, .szPage = pTsdb->pVnode->config.tsdbPageSize};




3,912✔





1415



        conf.file[0] = *pSttLevel->fobjArr->data[i]->f;




3,912✔





1416












1417



        const char* pName = pSttLevel->fobjArr->data[i]->fname;




3,912✔





1418



        code = tsdbSttFileReaderOpen(pName, &conf, &pIter->pReader);




3,912✔





1419



        if (code != TSDB_CODE_SUCCESS) {




3,912✔





1420



          tsdbError("open stt file reader error. file:%s, code %s, %s", pName, tstrerror(code), pstr);




×







1421



          continue;




×







1422



        }










1423



      }










1424












1425



      if (pIter->pBlockLoadInfo == NULL) {




3,912✔





1426



        code = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols, &pIter->pBlockLoadInfo);




3,912✔





1427



        if (code != TSDB_CODE_SUCCESS) {




3,912✔





1428



          tsdbError("failed to create block load info, code: out of memory, %s", pstr);




×







1429



          continue;




×







1430



        }










1431



      }










1432












1433



      // load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file










1434



      TStatisBlkArray* pStatisBlkArray = NULL;




3,912✔





1435



      code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray**)&pStatisBlkArray);




3,912✔





1436



      if (code != TSDB_CODE_SUCCESS) {




3,912✔





1437



        tsdbError("failed to load stt block statistics, code:%s, %s", tstrerror(code), pstr);




×







1438



        continue;




×







1439



      }










1440












1441



      // extract rows from each stt file one-by-one










1442



      STsdbReader* pReader = pConf->pReader;




3,912✔





1443



      int32_t      numOfTables = tSimpleHashGetSize(pReader->status.pTableMap);




3,912✔





1444



      uint64_t*    pUidList = pReader->status.uidList.tableUidList;




3,912✔





1445



      int32_t      n = 0;




3,912✔





1446



      code = getNumOfRowsInSttBlock(pIter->pReader, pIter->pBlockLoadInfo, pStatisBlkArray, pConf->suid, pUidList,




3,912✔





1447



                                    numOfTables, &n);










1448



      numOfRows += n;




3,912✔





1449



      if (code) {




3,912✔





1450



        tsdbError("%s failed to get rows in stt blocks, code:%s", pstr, tstrerror(code));




×







1451



      }










1452



    }










1453



  }










1454












1455



_end:




3,912✔





1456



  if (code != TSDB_CODE_SUCCESS) {




30,317✔





1457



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1458



  }










1459



  return numOfRows;




30,317✔





1460



}










1461












1462



static bool overlapHelper(const STimeWindow* pLeft, TSKEY minKey, TSKEY maxKey) {




5,358,480✔





1463



  return (pLeft != NULL) && (pLeft->ekey >= minKey) && (pLeft->skey <= maxKey);




5,358,480✔





1464



}










1465












1466



static bool overlapWithTimeWindow(STimeWindow* p1, STimeWindow* pQueryWindow, STableBlockScanInfo* pBlockScanInfo,




14,477,217✔





1467



                                  int32_t order) {










1468



  SIterInfo*   pMemIter = NULL;




14,477,217✔





1469



  SIterInfo*   pIMemIter = NULL;




14,477,217✔





1470



  STbData*     pTbData = NULL;




14,477,217✔





1471



  STimeWindow* pFileWin = NULL;




14,477,217✔





1472












1473



  if (p1 == NULL || pQueryWindow == NULL) {




14,477,217✔





1474



    return false;




40✔





1475



  }










1476












1477



  // overlap with query window










1478



  if (!(p1->skey >= pQueryWindow->skey && p1->ekey <= pQueryWindow->ekey)) {




14,479,133✔





1479



    return true;




147,240✔





1480



  }










1481












1482



  if (pBlockScanInfo == NULL) {




14,329,223✔





1483



    return false;




×







1484



  }










1485












1486



  pMemIter = &pBlockScanInfo->iter;




14,329,223✔





1487



  pIMemIter = &pBlockScanInfo->iiter;




14,329,853✔





1488












1489



  // overlap with mem data










1490



  if (pMemIter->hasVal) {




14,326,553✔





1491



    pTbData = pMemIter->iter->pTbData;




5,269,936✔





1492



    if (overlapHelper(p1, pTbData->minKey, pTbData->maxKey)) {




5,268,632✔





1493



      return true;




5,025,240✔





1494



    }










1495



  }










1496












1497



  // overlap with imem data










1498



  if (pIMemIter->hasVal) {




9,305,979✔





1499



    pTbData = pIMemIter->iter->pTbData;




82,151✔





1500



    if (overlapHelper(p1, pTbData->minKey, pTbData->maxKey)) {




82,151✔





1501



      return true;




52,000✔





1502



    }










1503



  }










1504












1505



  // overlap with data file block










1506



  pFileWin = &pBlockScanInfo->filesetWindow;




9,256,587✔





1507



  if ((taosArrayGetSize(pBlockScanInfo->pBlockIdxList) > 0) && overlapHelper(p1, pFileWin->skey, pFileWin->ekey)) {




9,256,587✔





1508



    return true;




×







1509



  }










1510












1511



  // overlap with deletion skyline










1512



  SBrinRecord record = {.firstKey = p1->skey, .lastKey = p1->ekey};




9,254,507✔





1513



  if (overlapWithDelSkylineWithoutVer(pBlockScanInfo, &record, order)) {




9,255,305✔





1514



    return true;




60,554✔





1515



  }










1516












1517



  return false;




9,191,367✔





1518



}










1519












1520



static int32_t sortUidComparFn(const void* p1, const void* p2) {




21,514,518✔





1521



  const SSttKeyRange* px1 = p1;




21,514,518✔





1522



  const SSttKeyRange* px2 = p2;




21,514,518✔





1523












1524



  int32_t ret = tRowKeyCompare(&px1->skey, &px2->skey);




21,514,518✔





1525



  return ret;




21,521,690✔





1526



}










1527












1528



bool isCleanSttBlock(SArray* pKeyRangeList, STimeWindow* pQueryWindow, STableBlockScanInfo* pScanInfo, int32_t order) {




26,802,718✔





1529



  int32_t       num = 0;




26,802,718✔





1530



  SSttKeyRange* pRange = NULL;




26,802,718✔





1531



  STimeWindow   w;




26,802,718✔





1532












1533



  num = taosArrayGetSize(pKeyRangeList);




26,805,466✔





1534



  if (num == 0) {




26,804,650✔





1535



    return false;




12,336,395✔





1536



  }










1537












1538



  // check if it overlap with del skyline










1539



  taosArraySort(pKeyRangeList, sortUidComparFn);




14,468,255✔





1540












1541



  pRange = taosArrayGet(pKeyRangeList, 0);




14,465,585✔





1542



  if (pRange == NULL) {




14,468,131✔





1543



    return false;




×







1544



  }










1545












1546



  w = (STimeWindow){.skey = pRange->skey.ts, .ekey = pRange->ekey.ts};




14,468,131✔





1547



  if (overlapWithTimeWindow(&w, pQueryWindow, pScanInfo, order)) {




14,462,915✔





1548



    return false;




5,277,862✔





1549



  }










1550












1551



  for (int32_t i = 0; i < num - 1; ++i) {




9,192,019✔





1552



    SSttKeyRange* p1 = taosArrayGet(pKeyRangeList, i);




3,965,532✔





1553



    SSttKeyRange* p2 = taosArrayGet(pKeyRangeList, i + 1);




3,966,836✔





1554



    if (p1 == NULL || p2 == NULL) {




3,966,836✔





1555



      return false;




×







1556



    }










1557












1558



    if (p1->ekey.ts >= p2->skey.ts) {




3,966,836✔





1559



      return false;




3,953,248✔





1560



    }










1561












1562



    w = (STimeWindow){.skey = p2->skey.ts, .ekey = p2->ekey.ts};




13,588✔





1563



    bool overlap = overlapWithTimeWindow(&w, pQueryWindow, pScanInfo, order);




13,588✔





1564



    if (overlap) {




13,588✔





1565



      return false;




×







1566



    }










1567



  }










1568












1569



  return true;




5,226,487✔





1570



}










1571












1572



static bool doCheckDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord,




169,184✔





1573



                                    int32_t startIndex) {










1574



  size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline);




169,184✔





1575












1576



  for (int32_t i = startIndex; i < num; i += 1) {




172,852✔





1577



    TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i);




172,852✔





1578



    if (p == NULL) {




172,852✔





1579



      return false;




×







1580



    }










1581












1582



    if (p->ts >= pRecord->firstKey.key.ts && p->ts <= pRecord->lastKey.key.ts) {




172,852✔





1583



      if (p->version >= pRecord->minVer) {




163,128✔





1584



        return true;




162,080✔





1585



      }










1586



    } else if (p->ts < pRecord->firstKey.key.ts) {  // p->ts < pBlock->minKey.ts




9,724✔





1587



      if (p->version >= pRecord->minVer) {




6,056✔





1588



        if (i < num - 1) {




3,436✔





1589



          TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1);




3,436✔





1590



          if (pnext == NULL) {




3,436✔





1591



            return false;




×







1592



          }










1593












1594



          if (pnext->ts >= pRecord->firstKey.key.ts) {




3,436✔





1595



            return true;




3,436✔





1596



          }










1597



        } else {  // it must be the last point










1598



          if (!(p->version == 0)) {




×







1599



            tsdbError("unexpected version:%" PRId64, p->version);




×







1600



          }










1601



        }










1602



      }










1603



    } else {  // (p->ts > pBlock->maxKey.ts) {










1604



      return false;




3,668✔





1605



    }










1606



  }










1607












1608



  return false;




×







1609



}










1610












1611



static bool doCheckDatablockOverlapWithoutVersion(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord,




60,554✔





1612



                                                  int32_t startIndex) {










1613



  size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline);




60,554✔





1614












1615



  for (int32_t i = startIndex; i < num; i += 1) {




60,554✔





1616



    TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i);




60,554✔





1617



    if (p == NULL) {




60,554✔





1618



      return false;




×







1619



    }










1620












1621



    if (p->ts >= pRecord->firstKey.key.ts && p->ts <= pRecord->lastKey.key.ts) {




60,554✔





1622



      return true;




60,554✔





1623



    } else if (p->ts < pRecord->firstKey.key.ts) {  // p->ts < pBlock->minKey.ts




×







1624



      if (i < num - 1) {




×







1625



        TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1);




×







1626



        if (pnext == NULL) {




×







1627



          return false;




×







1628



        }










1629












1630



        if (pnext->ts >= pRecord->firstKey.key.ts) {




×







1631



          return true;




×







1632



        }










1633



      }










1634



    } else {  // (p->ts > pBlock->maxKey.ts) {










1635



      return false;




×







1636



    }










1637



  }










1638












1639



  return false;




×







1640



}










1641












1642



bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, int32_t order) {




131,796,555✔





1643



  if (pBlockScanInfo->delSkyline == NULL || (taosArrayGetSize(pBlockScanInfo->delSkyline) == 0)) {




131,796,555✔





1644



    return false;




131,305,559✔





1645



  }










1646












1647



  // ts is not overlap










1648



  TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0);




523,424✔





1649



  TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline);




523,424✔





1650



  if (pFirst == NULL || pLast == NULL) {




523,424✔





1651



    return false;




×







1652



  }










1653












1654



  if (pRecord->firstKey.key.ts > pLast->ts || pRecord->lastKey.key.ts < pFirst->ts) {




523,424✔





1655



    return false;




354,240✔





1656



  }










1657












1658



  // version is not overlap










1659



  if (ASCENDING_TRAVERSE(order)) {




169,184✔





1660



    return doCheckDatablockOverlap(pBlockScanInfo, pRecord, pBlockScanInfo->fileDelIndex);




85,480✔





1661



  } else {










1662



    int32_t index = pBlockScanInfo->fileDelIndex;




83,704✔





1663



    while (1) {




73,884✔





1664



      TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, index);




157,588✔





1665



      if (p == NULL) {




157,588✔





1666



        return false;




×







1667



      }










1668












1669



      if (p->ts > pRecord->firstKey.key.ts && index > 0) {




157,588✔





1670



        index -= 1;




73,884✔





1671



      } else {  // find the first point that is smaller than the minKey.ts of dataBlock.










1672



        if (p->ts == pRecord->firstKey.key.ts && p->version < pRecord->maxVer && index > 0) {




83,704✔





1673



          index -= 1;




9,820✔





1674



        }










1675



        break;




83,704✔





1676



      }










1677



    }










1678












1679



    return doCheckDatablockOverlap(pBlockScanInfo, pRecord, index);




83,704✔





1680



  }










1681



}










1682












1683



bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, int32_t order) {




9,256,649✔





1684



  if (pBlockScanInfo->delSkyline == NULL || (taosArrayGetSize(pBlockScanInfo->delSkyline) == 0)) {




9,256,649✔





1685



    return false;




9,193,425✔





1686



  }










1687












1688



  // ts is not overlap










1689



  TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0);




60,554✔





1690



  TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline);




60,554✔





1691



  if (pFirst == NULL || pLast == NULL) {




60,554✔





1692



    return false;




×







1693



  }










1694












1695



  if (pRecord->firstKey.key.ts > pLast->ts || pRecord->lastKey.key.ts < pFirst->ts) {




60,554✔





1696



    return false;




×







1697



  }










1698












1699



  // version is not overlap










1700



  if (ASCENDING_TRAVERSE(order)) {




60,554✔





1701



    return doCheckDatablockOverlapWithoutVersion(pBlockScanInfo, pRecord, pBlockScanInfo->fileDelIndex);




60,554✔





1702



  } else {










1703



    int32_t index = pBlockScanInfo->fileDelIndex;




×







1704



    while (1) {




×







1705



      TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, index);




×







1706



      if (p == NULL) {




×







1707



        return false;




×







1708



      }










1709












1710



      if (p->ts > pRecord->firstKey.key.ts && index > 0) {




×







1711



        index -= 1;




×







1712



      } else {  // find the first point that is smaller than the minKey.ts of dataBlock.










1713



        if (p->ts == pRecord->firstKey.key.ts && index > 0) {




×







1714



          index -= 1;




×







1715



        }










1716



        break;




×







1717



      }










1718



    }










1719












1720



    return doCheckDatablockOverlapWithoutVersion(pBlockScanInfo, pRecord, index);




×







1721



  }










1722



}
























    

  • 

    •