taosdata / TDengine / #4838

coverage: 71.256% (+12.3%) from 58.963%
#4838

push

travis-ci

web-flow 
test: adjust source list (#33506)

243241 of 341361 relevant lines covered (71.26%)

281946921.97 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

86.75
/source/dnode/vnode/src/tsdb/tsdbRead2.c
1 
/*
2 
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3 
 *
4 
 * This program is free software: you can use, redistribute, and/or modify
5 
 * it under the terms of the GNU Affero General Public License, version 3
6 
 * or later ("AGPL"), as published by the Free Software Foundation.
7 
 *
8 
 * This program is distributed in the hope that it will be useful, but WITHOUT
9 
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 
 * FITNESS FOR A PARTICULAR PURPOSE.
11 
 *
12 
 * You should have received a copy of the GNU Affero General Public License
13 
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
14 
 */
15 









16



#include "osDef.h"










17



#include "tdb.h"










18



#include "tsdb.h"










19



#include "tsdbDataFileRW.h"










20



#include "tsdbFS2.h"










21



#include "tsdbMerge.h"










22



#include "tsdbReadUtil.h"










23



#include "tsdbUtil2.h"










24



#include "tsimplehash.h"










25



#include "vnode.h"










26












27



#define ASCENDING_TRAVERSE(o)       (o == TSDB_ORDER_ASC)










28



#define getCurrentKeyInSttBlock(_r) (&((_r)->currentKey))










29



#define tColRowGetKeyDeepCopy(_pBlock, _irow, _slotId, _pKey)             \










30



  do {                                                                    \










31



    (_pKey)->ts = (_pBlock)->aTSKEY[(_irow)];                             \










32



    (_pKey)->numOfPKs = 0;                                                \










33



    if ((_slotId) != -1) {                                                \










34



      code = tColRowGetPriamyKeyDeepCopy(_pBlock, _irow, _slotId, _pKey); \










35



      TSDB_CHECK_CODE(code, lino, _end);                                  \










36



    }                                                                     \










37



  } while (0)










38












39



#define outOfTimeWindow(_ts, _window) (((_ts) > (_window)->ekey) || ((_ts) < (_window)->skey))










40












41



typedef struct {










42



  bool overlapWithNeighborBlock;










43



  bool hasDupTs;










44



  bool overlapWithDelInfo;










45



  bool overlapWithSttBlock;










46



  bool overlapWithKeyInBuf;










47



  bool partiallyRequired;










48



  bool moreThanCapcity;










49



} SDataBlockToLoadInfo;










50












51



static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInfo** pInfo, const char* idStr);










52



static int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity,










53



                                         STsdbReader* pReader);










54



static int32_t getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader, TSDBROW** pRes);










55



static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, SRowKey* pKey,










56



                                       STsdbReader* pReader);










57



static int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo,










58



                                     SRowMerger* pMerger, int32_t pkSrcSlot, SVersionRange* pVerRange, const char* id);










59



static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey* pCurKey, SArray* pDelList,










60



                                STsdbReader* pReader);










61



static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow,










62



                                    STableBlockScanInfo* pScanInfo);










63



static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData,










64



                                        int32_t rowIndex);










65



static void    setComposedBlockFlag(STsdbReader* pReader, bool composed);










66



static int32_t hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t ver, int32_t order,










67



                              SVersionRange* pVerRange, bool hasPk, bool* dropped);










68












69



static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, SRowKey* pKey, uint64_t uid, SIterInfo* pIter, SArray* pDelList,










70



                                        TSDBROW* pResRow, STsdbReader* pReader, bool* freeTSRow);










71



static int32_t doMergeMemIMemRows(TSDBROW* pRow, SRowKey* pRowKey, TSDBROW* piRow, SRowKey* piRowKey,










72



                                  STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, SRow** pTSRow);










73



static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, SRowKey* pKey,










74



                                     STsdbReader* pReader);










75



static int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo,










76



                                    STsdbReader* pReader);










77












78



static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SReadCostSummary* pCost);










79



static void getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idstr,










80



                                int8_t* pLevel, STsdb** pTsdb);










81



static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond);










82



static int32_t       doBuildDataBlock(STsdbReader* pReader);










83



static int32_t       getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader, TSDBKEY* key);










84



static bool          hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo);










85



static bool          hasDataInSttBlock(STableBlockScanInfo* pInfo);










86



static int32_t       initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter);










87



static int32_t       getInitialDelIndex(const SArray* pDelSkyline, int32_t order);










88



static int32_t       resetTableListIndex(SReaderStatus* pStatus, const char* id);










89



static void          getMemTableTimeRange(STsdbReader* pReader, int64_t* pMaxKey, int64_t* pMinKey);










90



static void          updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInfo* pBlockScanInfo);










91



static int32_t       buildFromPreFilesetBuffer(STsdbReader* pReader);










92












93



static void resetPreFilesetMemTableListIndex(SReaderStatus* pStatus);










94



int32_t     tsdbReaderSuspend2(STsdbReader* pReader);










95












96



FORCE_INLINE int32_t pkCompEx(SRowKey* p1, SRowKey* p2) {




1,001,715✔





97



  if (p2 == NULL) {




2,147,483,647✔





98



    return 1;




2,147,483,647✔





99



  }










100












101



  if (p1 == NULL) {




2,147,483,647✔





102



    return -1;




×







103



  }










104












105



  if (p1->ts < p2->ts) {




2,147,483,647✔





106



    return -1;




2,147,483,647✔





107



  } else if (p1->ts > p2->ts) {




2,147,483,647✔





108



    return 1;




2,147,483,647✔





109



  }










110












111



  if (p1->numOfPKs == 0) {




2,147,483,647✔





112



    return 0;




2,147,483,647✔





113



  } else {










114



    return tRowKeyCompare(p1, p2);




1,997,108,636✔





115



  }










116



}










117












118



static int32_t tColRowGetPriamyKeyDeepCopy(SBlockData* pBlock, int32_t irow, int32_t slotId, SRowKey* pKey) {




2,147,483,647✔





119



  int32_t   code = TSDB_CODE_SUCCESS;




2,147,483,647✔





120



  int32_t   lino = 0;




2,147,483,647✔





121



  SColData* pColData = NULL;




2,147,483,647✔





122



  SColVal   cv;




2,147,483,647✔





123












124



  TSDB_CHECK_CONDITION((pBlock != NULL) && (pBlock->aColData != NULL), code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





125



  TSDB_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





126












127



  pColData = &pBlock->aColData[slotId];




2,147,483,647✔





128












129



  code = tColDataGetValue(pColData, irow, &cv);




2,147,483,647✔





130



  TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





131












132



  pKey->numOfPKs = 1;




2,147,483,647✔





133



  pKey->pks[0].type = cv.value.type;




2,147,483,647✔





134












135



  if (IS_NUMERIC_TYPE(cv.value.type)) {




2,147,483,647✔





136



    valueCloneDatum(pKey->pks, &cv.value, cv.value.type);




2,147,483,647✔





137



  } else {










138



    pKey->pks[0].nData = cv.value.nData;




1,544,054,044✔





139



    TAOS_MEMCPY(pKey->pks[0].pData, cv.value.pData, cv.value.nData);




1,544,434,382✔





140



  }










141












142



_end:




2,147,483,647✔





143



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





144



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







145



  }










146



  return code;




2,147,483,647✔





147



}










148












149



// for test purpose, todo remove it










150



static int32_t tGetPrimaryKeyIndex(uint8_t* p, SPrimaryKeyIndex* index) {




686,909,755✔





151



  int32_t n = 0;




686,909,755✔





152



  n += tGetI8(p + n, &index->type);




686,909,755✔





153



  n += tGetU32v(p + n, &index->offset);




686,961,279✔





154



  return n;




687,034,179✔





155



}










156












157



static int32_t tRowGetPrimaryKeyDeepCopy(SRow* pRow, SRowKey* pKey) {




686,826,585✔





158



  int32_t          code = TSDB_CODE_SUCCESS;




686,826,585✔





159



  int32_t          lino = 0;




686,826,585✔





160



  SPrimaryKeyIndex indices[TD_MAX_PK_COLS];




686,826,585✔





161



  uint8_t*         data = NULL;




686,884,495✔





162



  int32_t          len = 0;




686,884,495✔





163












164



  TSDB_CHECK_NULL(pRow, code, lino, _end, TSDB_CODE_INVALID_PARA);




686,884,495✔





165



  if (pRow->numOfPKs > 0) {




686,884,495✔





166



    TSDB_CHECK_NULL(pRow->data, code, lino, _end, TSDB_CODE_INVALID_PARA);




687,005,312✔





167



    TSDB_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA);




687,013,793✔





168



  }










169












170



  data = pRow->data;




686,992,314✔





171



  for (int32_t i = 0; i < pRow->numOfPKs; i++) {




1,373,965,338✔





172



    data += tGetPrimaryKeyIndex(data, &indices[i]);




686,997,812✔





173



  }










174












175



  // primary keys










176



  for (int32_t i = 0; i < pRow->numOfPKs; i++) {




1,373,808,335✔





177



    pKey->pks[i].type = indices[i].type;




686,985,357✔





178












179



    uint8_t* tdata = data + indices[i].offset;




686,957,098✔





180



    if (pRow->flag >> 4) {




686,941,193✔





181



      tdata += tGetI16v(tdata, NULL);




451,683,400✔





182



    }










183












184



    if (IS_VAR_DATA_TYPE(indices[i].type)) {




686,956,939✔





185



      tdata += tGetU32v(tdata, &pKey->pks[i].nData);




309,796,870✔





186



      TAOS_MEMCPY(pKey->pks[i].pData, tdata, pKey->pks[i].nData);




154,911,140✔





187



    } else {










188



      valueSetDatum(pKey->pks + i, indices[i].type, data + indices[i].offset, tDataTypes[pKey->pks[i].type].bytes);




532,106,978✔





189



    }










190



  }










191












192



_end:




686,892,786✔





193



  if (code != TSDB_CODE_SUCCESS) {




686,896,970✔





194



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







195



  }










196



  return code;




686,896,970✔





197



}










198












199



static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList,




1,891,067,323✔





200



                                   int32_t numOfCols) {










201



  int32_t code = TSDB_CODE_SUCCESS;




1,891,067,323✔





202



  int32_t lino = 0;




1,891,067,323✔





203












204



  TSDB_CHECK_NULL(pSupInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,891,067,323✔





205



  if (numOfCols > 0) {




1,891,067,323✔





206



    TSDB_CHECK_NULL(pSlotIdList, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,891,004,250✔





207



    TSDB_CHECK_NULL(pCols, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,891,004,250✔





208



  }










209












210



  pSupInfo->pk.pk = 0;




1,891,067,323✔





211



  pSupInfo->numOfPks = 0;




1,891,282,244✔





212



  pSupInfo->pkSrcSlot = -1;




1,890,742,291✔





213



  pSupInfo->pkDstSlot = -1;




1,890,137,465✔





214



  pSupInfo->smaValid = true;




1,890,819,637✔





215



  pSupInfo->numOfCols = numOfCols;




1,891,198,968✔





216












217



  pSupInfo->colId = taosMemoryCalloc(numOfCols, sizeof(int16_t) * 2 + POINTER_BYTES);




1,889,527,708✔





218



  TSDB_CHECK_NULL(pSupInfo->colId, code, lino, _end, terrno);




1,890,818,176✔





219












220



  pSupInfo->slotId = (int16_t*)((char*)pSupInfo->colId + (sizeof(int16_t) * numOfCols));




1,890,069,545✔





221



  pSupInfo->buildBuf = (char**)((char*)pSupInfo->slotId + (sizeof(int16_t) * numOfCols));




1,890,445,916✔





222



  for (int32_t i = 0; i < numOfCols; ++i) {




2,147,483,647✔





223



    pSupInfo->colId[i] = pCols[i].colId;




2,147,483,647✔





224



    pSupInfo->slotId[i] = pSlotIdList[i];




2,147,483,647✔





225












226



    if (pCols[i].pk) {




2,147,483,647✔





227



      pSupInfo->pk = pCols[i];




59,709,586✔





228



      pSupInfo->pkSrcSlot = i - 1;




59,648,995✔





229



      pSupInfo->pkDstSlot = pSlotIdList[i];




59,491,082✔





230



      pSupInfo->numOfPks += 1;




59,666,338✔





231



    }










232












233



    if (IS_VAR_DATA_TYPE(pCols[i].type)) {




2,147,483,647✔





234



      pSupInfo->buildBuf[i] = taosMemoryMalloc(pCols[i].bytes);




884,839,396✔





235



      if (pSupInfo->buildBuf[i] == NULL) {




881,179,624✔





236



        tsdbError("failed to prepare memory for set columnId slot list, size:%d, code:%s", pCols[i].bytes,




×







237



                  tstrerror(terrno));










238



      }










239



      TSDB_CHECK_NULL(pSupInfo->buildBuf[i], code, lino, _end, terrno);




880,881,953✔





240



    } else {










241



      pSupInfo->buildBuf[i] = NULL;




2,147,483,647✔





242



    }










243



  }










244












245



_end:




1,891,236,141✔





246



  if (code != TSDB_CODE_SUCCESS) {




1,891,236,141✔





247



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







248



  }










249



  return code;




1,891,024,934✔





250



}










251












252



static int32_t updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo) {




1,830,156,237✔





253



  int32_t   code = TSDB_CODE_SUCCESS;




1,830,156,237✔





254



  int32_t   lino = 0;




1,830,156,237✔





255



  int32_t   i = 0, j = 0;




1,830,156,237✔





256



  STColumn* pTCol = NULL;




1,830,156,237✔





257












258



  TSDB_CHECK_NULL(pSchema, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,830,156,237✔





259



  TSDB_CHECK_NULL(pSupInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,830,156,237✔





260












261



  while (i < pSchema->numOfCols && j < pSupInfo->numOfCols) {




2,147,483,647✔





262



    pTCol = &pSchema->columns[i];




2,147,483,647✔





263



    if (pTCol->colId == pSupInfo->colId[j]) {




2,147,483,647✔





264



      if (!IS_BSMA_ON(pTCol) && (PRIMARYKEY_TIMESTAMP_COL_ID != pTCol->colId)) {




2,147,483,647✔





265



        pSupInfo->smaValid = false;




90,025,972✔





266



        goto _end;




88,913,800✔





267



      }










268












269



      i += 1;




2,147,483,647✔





270



      j += 1;




2,147,483,647✔





271



    } else if (pTCol->colId < pSupInfo->colId[j]) {  // do nothing




2,147,483,647✔





272



      i += 1;




2,147,483,647✔





273



    } else {










274



      code = TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER;




49,150✔





275



      TSDB_CHECK_CODE(code, lino, _end);




49,150✔





276



    }










277



  }










278












279



_end:




1,738,987,021✔





280



  if (code != TSDB_CODE_SUCCESS) {




1,827,949,971✔





281



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




49,150✔





282



  }










283



  return code;




1,830,301,866✔





284



}










285












286



static bool isEmptyQueryTimeWindow(STimeWindow* pWindow) {




2,147,483,647✔





287



  return (pWindow == NULL) || (pWindow->skey > pWindow->ekey);




2,147,483,647✔





288



}










289












290



// Update the query time window according to the data time to live(TTL) information, in order to avoid to return










291



// the expired data to client, even it is queried already.










292



static int32_t updateQueryTimeWindow(STsdb* pTsdb, STimeWindow* pWindow, STimeWindow* out) {




2,018,520,634✔





293



  int32_t code = TSDB_CODE_SUCCESS;




2,018,520,634✔





294



  int32_t lino = 0;




2,018,520,634✔





295



  int64_t earlyTs = 0;




2,018,520,634✔





296












297



  TSDB_CHECK_NULL(pTsdb, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,018,520,634✔





298



  TSDB_CHECK_NULL(pWindow, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,018,520,634✔





299



  TSDB_CHECK_NULL(out, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,018,520,634✔





300












301



  earlyTs = tsdbGetEarliestTs(pTsdb);




2,018,520,634✔





302



  *out = *pWindow;




2,018,211,372✔





303



  if (out->skey < earlyTs) {




2,018,619,795✔





304



    out->skey = earlyTs;




1,723,991,978✔





305



  }










306












307



_end:




293,211,788✔





308



  if (code != TSDB_CODE_SUCCESS) {




2,017,254,151✔





309



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







310



  }










311



  return code;




2,018,303,539✔





312



}










313












314



// init file iterator










315



static int32_t initFilesetIterator(SFilesetIter* pIter, TFileSetArray* pFileSetArray, STsdbReader* pReader) {




1,843,088,542✔





316



  int32_t             code = TSDB_CODE_SUCCESS;




1,843,088,542✔





317



  int32_t             lino = 0;




1,843,088,542✔





318



  SBlockLoadSuppInfo* pInfo = NULL;




1,843,088,542✔





319



  SSttBlockReader*    pSttReader = NULL;




1,843,088,542✔





320



  size_t              numOfFileset = 0;




1,843,088,542✔





321



  bool                asc = false;




1,843,088,542✔





322












323



  TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,843,088,542✔





324



  TSDB_CHECK_NULL(pFileSetArray, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,843,088,542✔





325



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,843,088,542✔





326












327



  pInfo = &pReader->suppInfo;




1,843,088,542✔





328



  numOfFileset = TARRAY2_SIZE(pFileSetArray);




1,843,508,439✔





329



  asc = ASCENDING_TRAVERSE(pReader->info.order);




1,843,700,760✔





330












331



  pIter->index = asc ? -1 : numOfFileset;




1,842,556,734✔





332



  pIter->order = pReader->info.order;




1,842,219,920✔





333



  pIter->pFilesetList = pFileSetArray;




1,843,032,923✔





334



  pIter->numOfFiles = numOfFileset;




1,842,626,365✔





335












336



  if (pIter->pSttBlockReader == NULL) {




1,842,462,731✔





337



    pIter->pSttBlockReader = taosMemoryCalloc(1, sizeof(struct SSttBlockReader));




1,716,329,870✔





338



    if (pIter->pSttBlockReader == NULL) {




1,714,687,188✔





339



      tsdbError("failed to prepare the last block iterator, since:%s %s", tstrerror(terrno), pReader->idStr);




×







340



    }










341



    TSDB_CHECK_NULL(pIter->pSttBlockReader, code, lino, _end, terrno);




1,715,051,341✔





342



  }










343












344



  pSttReader = pIter->pSttBlockReader;




1,842,675,178✔





345



  pSttReader->order = pReader->info.order;




1,843,326,272✔





346



  pSttReader->window = pReader->info.window;




1,842,539,838✔





347



  pSttReader->verRange = pReader->info.verRange;




1,843,033,149✔





348



  pSttReader->numOfPks = pReader->suppInfo.numOfPks;




1,842,415,487✔





349



  pSttReader->uid = 0;




1,842,952,960✔





350












351



  tMergeTreeClose(&pSttReader->mergeTree);




1,843,285,663✔





352



  code = initRowKey(&pSttReader->currentKey, INT64_MIN, pInfo->numOfPks, pInfo->pk.type, pInfo->pk.bytes, asc);




1,843,199,610✔





353



  if (code != TSDB_CODE_SUCCESS) {




1,841,320,465✔





354



    tsdbError("failed init row key, %s", pReader->idStr);




×







355



  } else {










356



    tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, pReader->idStr);




1,841,320,465✔





357



  }










358



  TSDB_CHECK_CODE(code, lino, _end);




1,842,799,430✔





359












360



_end:




1,842,799,430✔





361



  if (code != TSDB_CODE_SUCCESS) {




1,842,799,430✔





362



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







363



  }










364



  return code;




1,842,131,404✔





365



}










366












367



static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bool* hasNext) {




2,147,483,647✔





368



  int32_t           code = TSDB_CODE_SUCCESS;




2,147,483,647✔





369



  int32_t           lino = 0;




2,147,483,647✔





370



  bool              asc = false;




2,147,483,647✔





371



  int32_t           step = 0;




2,147,483,647✔





372



  SReadCostSummary* pCost = NULL;




2,147,483,647✔





373



  STFileObj**       pFileObj = NULL;




2,147,483,647✔





374












375



  TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





376



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





377



  TSDB_CHECK_NULL(hasNext, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





378












379



  asc = ASCENDING_TRAVERSE(pIter->order);




2,147,483,647✔





380



  step = asc ? 1 : -1;




2,147,483,647✔





381



  *hasNext = false;




2,147,483,647✔





382












383



  pIter->index += step;




2,147,483,647✔





384



  if ((asc && pIter->index >= pIter->numOfFiles) || ((!asc) && pIter->index < 0)) {




2,147,483,647✔





385



    *hasNext = false;




418,233,726✔





386



    goto _end;




418,293,197✔





387



  }










388












389



  pCost = &pReader->cost;




2,147,483,647✔





390












391



  TSDB_CHECK_NULL(pIter->pSttBlockReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





392



  pIter->pSttBlockReader->uid = 0;




2,147,483,647✔





393



  tMergeTreeClose(&pIter->pSttBlockReader->mergeTree);




2,147,483,647✔





394



  destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost);




2,147,483,647✔





395












396



  pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES);




2,147,483,647✔





397



  TSDB_CHECK_NULL(pReader->status.pLDataIterArray, code, lino, _end, terrno);




2,147,483,647✔





398












399



  // check file the time range of coverage










400



  STimeWindow win = {0};




2,147,483,647✔





401












402



  while (1) {




3,783,092✔





403



    if (pReader->pFileReader != NULL) {




2,147,483,647✔





404



      tsdbDataFileReaderClose(&pReader->pFileReader);




132,988,161✔





405



    }










406












407



    TSDB_CHECK_CONDITION(pIter->index < pIter->pFilesetList->size, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);




2,147,483,647✔





408



    pReader->status.pCurrentFileset = pIter->pFilesetList->data[pIter->index];




2,147,483,647✔





409












410



    pFileObj = pReader->status.pCurrentFileset->farr;




2,147,483,647✔





411



    if (pFileObj[0] != NULL || pFileObj[3] != NULL) {




2,147,483,647✔





412



      SDataFileReaderConfig conf = {.tsdb = pReader->pTsdb, .szPage = pReader->pTsdb->pVnode->config.tsdbPageSize};




168,174,210✔





413












414



      const char* filesName[4] = {0};




168,200,902✔





415












416



      if (pFileObj[0] != NULL) {




168,202,712✔





417



        conf.files[0].file = *pFileObj[0]->f;




149,045,404✔





418



        conf.files[0].exist = true;




149,044,267✔





419



        filesName[0] = pFileObj[0]->fname;




149,044,267✔





420












421



        conf.files[1].file = *pFileObj[1]->f;




149,046,811✔





422



        conf.files[1].exist = true;




149,046,563✔





423



        filesName[1] = pFileObj[1]->fname;




149,046,563✔





424












425



        conf.files[2].file = *pFileObj[2]->f;




149,042,451✔





426



        conf.files[2].exist = true;




149,043,108✔





427



        filesName[2] = pFileObj[2]->fname;




149,043,108✔





428



      }










429












430



      if (pFileObj[3] != NULL) {




168,187,599✔





431



        conf.files[3].exist = true;




19,845,812✔





432



        conf.files[3].file = *pFileObj[3]->f;




19,845,812✔





433



        filesName[3] = pFileObj[3]->fname;




19,845,812✔





434



      }










435












436



      code = tsdbDataFileReaderOpen(filesName, &conf, &pReader->pFileReader);




168,191,588✔





437



      TSDB_CHECK_CODE(code, lino, _end);




168,197,295✔





438












439



      pReader->cost.headFileLoad += 1;




168,197,295✔





440



    }










441












442



    int32_t fid = pReader->status.pCurrentFileset->fid;




2,147,483,647✔





443



    tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &win.skey, &win.ekey);




2,147,483,647✔





444












445



    // current file are no longer overlapped with query time window, ignore remain files










446



    if ((asc && win.skey > pReader->info.window.ekey) || (!asc && win.ekey < pReader->info.window.skey)) {




2,147,483,647✔





447



      tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %s", pReader,




1,966,167✔





448



                pReader->info.window.skey, pReader->info.window.ekey, pReader->idStr);










449



      *hasNext = false;




1,966,167✔





450



      break;




1,966,167✔





451



    }










452












453



    if ((asc && (win.ekey < pReader->info.window.skey)) || ((!asc) && (win.skey > pReader->info.window.ekey))) {




2,147,483,647✔





454



      pIter->index += step;




6,004,626✔





455



      if ((asc && pIter->index >= pIter->numOfFiles) || ((!asc) && pIter->index < 0)) {




3,942,922✔





456



        *hasNext = false;




159,830✔





457



        break;




161,515✔





458



      }










459



      continue;




3,783,092✔





460



    }










461












462



    tsdbDebug("%p file found fid:%d for qrange:%" PRId64 "-%" PRId64 ", %s", pReader, fid, pReader->info.window.skey,




2,147,483,647✔





463



              pReader->info.window.ekey, pReader->idStr);










464












465



    *hasNext = true;




2,147,483,647✔





466



    break;




2,147,483,647✔





467



  }










468












469



_end:




2,147,483,647✔





470



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





471



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







472



  }










473



  return code;




2,147,483,647✔





474



}










475












476



bool shouldFreePkBuf(SBlockLoadSuppInfo* pSupp) {




2,147,483,647✔





477



  return (pSupp != NULL) && (pSupp->numOfPks > 0) && IS_VAR_DATA_TYPE(pSupp->pk.type);




2,147,483,647✔





478



}










479












480



int32_t resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool needFree, const char* id) {




2,147,483,647✔





481



  int32_t code = TSDB_CODE_SUCCESS;




2,147,483,647✔





482



  int32_t lino = 0;




2,147,483,647✔





483












484



  TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





485












486



  pIter->order = order;




2,147,483,647✔





487



  pIter->index = -1;




2,147,483,647✔





488



  pIter->numOfBlocks = 0;




2,147,483,647✔





489












490



  if (pIter->blockList == NULL) {




2,147,483,647✔





491



    pIter->blockList = taosArrayInit(4, sizeof(SFileDataBlockInfo));




1,714,968,226✔





492



    if (pIter->blockList == NULL) {




1,714,908,530✔





493



      tsdbError("%s failed to reset block iter, func:%s at line:%d code:%s", id, __func__, __LINE__, tstrerror(terrno));




×







494



    }










495



    TSDB_CHECK_NULL(pIter->blockList, code, lino, _end, terrno);




1,715,585,519✔





496



  } else {










497



    clearDataBlockIterator(pIter, needFree);




2,147,483,647✔





498



  }










499












500



_end:




2,147,483,647✔





501



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





502



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







503



  }










504



  return code;




2,147,483,647✔





505



}










506












507



static void initReaderStatus(SReaderStatus* pStatus) {




1,891,325,514✔





508



  if (pStatus == NULL) {




1,891,325,514✔





509



    return;




×







510



  }










511



  pStatus->pTableIter = NULL;




1,891,325,514✔





512



  pStatus->loadFromFile = true;




1,891,635,965✔





513



}










514












515



static int32_t createResBlock(SQueryTableDataCond* pCond, int32_t capacity, SSDataBlock** pResBlock) {




7,638,191✔





516



  int32_t      code = TSDB_CODE_SUCCESS;




7,638,191✔





517



  int32_t      lino = 0;




7,638,191✔





518



  SSDataBlock* pBlock = NULL;




7,638,191✔





519












520



  TSDB_CHECK_NULL(pCond, code, lino, _end, TSDB_CODE_INVALID_PARA);




7,637,111✔





521



  TSDB_CHECK_CONDITION(capacity >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA);




7,637,111✔





522



  TSDB_CHECK_NULL(pResBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);




7,637,111✔





523












524



  code = createDataBlock(&pBlock);




7,637,111✔





525



  TSDB_CHECK_CODE(code, lino, _end);




7,640,040✔





526












527



  for (int32_t i = 0; i < pCond->numOfCols; ++i) {




48,342,345✔





528



    SColumnInfoData colInfo = {0};




40,700,456✔





529



    colInfo.info = pCond->colList[i];




40,700,456✔





530



    code = blockDataAppendColInfo(pBlock, &colInfo);




40,698,607✔





531



    TSDB_CHECK_CODE(code, lino, _end);




40,702,305✔





532



  }










533












534



  code = blockDataEnsureCapacity(pBlock, capacity);




7,641,889✔





535



  TSDB_CHECK_CODE(code, lino, _end);




7,640,809✔





536












537



  *pResBlock = pBlock;




7,640,809✔





538



  pBlock = NULL;




7,640,809✔





539












540



_end:




7,640,809✔





541



  if (code != TSDB_CODE_SUCCESS) {




7,640,809✔





542



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







543



  }










544



  if (pBlock) {




7,640,809✔





545



    taosArrayDestroy(pBlock->pDataBlock);




×







546



    taosMemoryFreeClear(pBlock);




×







547



  }










548



  return code;




7,640,809✔





549



}










550












551



static int32_t tsdbInitReaderLock(STsdbReader* pReader) {




1,890,705,075✔





552



  int32_t code = TSDB_CODE_SUCCESS;




1,890,705,075✔





553



  int32_t lino = 0;




1,890,705,075✔





554












555



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,890,705,075✔





556












557



  code = taosThreadMutexInit(&pReader->readerMutex, NULL);




1,890,705,075✔





558



  tsdbTrace("tsdb/read: %p, post-init read mutex: %p, code: %d", pReader, &pReader->readerMutex, code);




1,890,315,961✔





559



  TSDB_CHECK_CODE(code, lino, _end);




1,890,382,166✔





560












561



_end:




1,890,382,166✔





562



  if (code != TSDB_CODE_SUCCESS) {




1,890,382,166✔





563



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







564



  }










565



  return code;




1,889,953,593✔





566



}










567












568



static int32_t tsdbUninitReaderLock(STsdbReader* pReader) {




1,891,461,159✔





569



  int32_t code = TSDB_CODE_SUCCESS;




1,891,461,159✔





570



  int32_t lino = 0;




1,891,461,159✔





571












572



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,891,461,159✔





573












574



  tsdbTrace("tsdb/read: %p, pre-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code);




1,891,461,159✔





575



  code = taosThreadMutexDestroy(&pReader->readerMutex);




1,891,461,159✔





576



  tsdbTrace("tsdb/read: %p, post-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code);




1,891,401,528✔





577



  TSDB_CHECK_CODE(code, lino, _end);




1,891,082,850✔





578












579



_end:




1,891,082,850✔





580



  if (code != TSDB_CODE_SUCCESS) {




1,891,082,850✔





581



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







582



  }










583



  return code;




1,891,460,082✔





584



}










585












586



static int32_t tsdbAcquireReader(STsdbReader* pReader) {




2,147,483,647✔





587



  int32_t code = TSDB_CODE_SUCCESS;




2,147,483,647✔





588



  int32_t lino = 0;




2,147,483,647✔





589












590



  TSDB_CHECK_CONDITION((pReader != NULL) && (pReader->idStr != NULL), code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





591












592



  tsdbTrace("tsdb/read: %s, pre-take read mutex: %p, code: %d", pReader->idStr, &pReader->readerMutex, code);




2,147,483,647✔





593



  code = taosThreadMutexLock(&pReader->readerMutex);




2,147,483,647✔





594



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





595



    tsdbError("tsdb/read:%p, failed to lock reader mutex, code:%s", pReader->idStr, tstrerror(code));




×







596



  } else {










597



    tsdbTrace("tsdb/read: %s, post-take read mutex: %p, code: %d", pReader->idStr, &pReader->readerMutex, code);




2,147,483,647✔





598



  }










599



  TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





600












601



_end:




2,147,483,647✔





602



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





603



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







604



  }










605



  return code;




2,147,483,647✔





606



}










607












608



static int32_t tsdbTryAcquireReader(STsdbReader* pReader) {




×







609



  int32_t code = TSDB_CODE_SUCCESS;




×







610



  int32_t lino = 0;




×







611












612



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




×







613












614



  code = taosThreadMutexTryLock(&pReader->readerMutex);




×







615



  if (code != TSDB_CODE_SUCCESS) {




×







616



    // Failing to acquire the lock is reasonable, not an error










617



    tsdbWarn("tsdb/read: %p, post-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code);




×







618



  } else {










619



    tsdbTrace("tsdb/read: %p, post-trytask read mutex: %p", pReader, &pReader->readerMutex);




×







620



  }










621



  return code;




×







622












623



_end:




×







624



  if (code != TSDB_CODE_SUCCESS) {




×







625



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







626



  }










627



  return code;




×







628



}










629












630



static int32_t tsdbReleaseReader(STsdbReader* pReader) {




2,147,483,647✔





631



  int32_t code = TSDB_CODE_SUCCESS;




2,147,483,647✔





632



  int32_t lino = 0;




2,147,483,647✔





633












634



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





635












636



  code = taosThreadMutexUnlock(&pReader->readerMutex);




2,147,483,647✔





637



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





638



    tsdbError("tsdb/read: %p post-untake read mutex:%p failed, code:%d", pReader, &pReader->readerMutex, code);




×







639



  } else {










640



    tsdbTrace("tsdb/read: %p, post-untake read mutex: %p", pReader, &pReader->readerMutex);




2,147,483,647✔





641



  }










642



  TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





643












644



_end:




2,147,483,647✔





645



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





646



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







647



  }










648



  return code;




2,147,483,647✔





649



}










650












651



void tsdbReleaseDataBlock2(void* p) {




399,096,352✔





652



  if (p == NULL) return;




399,096,352✔





653












654



  STsdbReader* pReader = (STsdbReader*)p;




399,096,352✔





655



  SReaderStatus* pStatus = &pReader->status;




399,096,352✔





656



  if (!pStatus->composedDataBlock) {




399,109,819✔





657



    (void)tsdbReleaseReader(pReader);




109,205,106✔





658



  }










659



}










660












661



static int32_t initResBlockInfo(SResultBlockInfo* pResBlockInfo, int64_t capacity, SSDataBlock* pResBlock,




1,890,838,742✔





662



                                SQueryTableDataCond* pCond, SBlockLoadSuppInfo* pSup) {










663



  int32_t      code = TSDB_CODE_SUCCESS;




1,890,838,742✔





664



  int32_t      lino = 0;




1,890,838,742✔





665



  SSDataBlock* p = NULL;




1,890,838,742✔





666












667



  TSDB_CHECK_NULL(pResBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,890,838,742✔





668



  TSDB_CHECK_CONDITION((pResBlockInfo->pResBlock != NULL) || (pSup != NULL), code, lino, _end, TSDB_CODE_INVALID_PARA);




1,890,838,742✔





669












670



  pResBlockInfo->capacity = capacity;




1,891,082,806✔





671



  pResBlockInfo->pResBlock = pResBlock;




1,890,308,851✔





672












673



  if (pResBlockInfo->pResBlock == NULL) {




1,890,846,674✔





674



    pResBlockInfo->freeBlock = true;




7,640,040✔





675



    code = createResBlock(pCond, pResBlockInfo->capacity, &pResBlockInfo->pResBlock);




7,640,040✔





676



    TSDB_CHECK_CODE(code, lino, _end);




7,641,889✔





677



  } else {










678



    pResBlockInfo->freeBlock = false;




1,882,114,386✔





679



  }










680












681



  if (pSup->numOfPks > 0) {




1,891,530,363✔





682



    p = pResBlockInfo->pResBlock;




59,569,677✔





683



    p->info.pks[0].type = pSup->pk.type;




59,664,270✔





684



    p->info.pks[1].type = pSup->pk.type;




59,641,769✔





685












686



    if (IS_VAR_DATA_TYPE(pSup->pk.type) && (p->info.pks[0].pData == NULL)) {




59,537,406✔





687



      p->info.pks[0].pData = taosMemoryCalloc(1, pSup->pk.bytes);




×







688



      TSDB_CHECK_NULL(p->info.pks[0].pData, code, lino, _end, terrno);




×







689












690



      p->info.pks[1].pData = taosMemoryCalloc(1, pSup->pk.bytes);




×







691



      TSDB_CHECK_NULL(p->info.pks[0].pData, code, lino, _end, terrno);




×







692












693



      p->info.pks[0].nData = pSup->pk.bytes;




×







694



      p->info.pks[1].nData = pSup->pk.bytes;




×







695



    }










696



  }










697












698



_end:




1,890,359,852✔





699



  if (code != TSDB_CODE_SUCCESS) {




1,890,359,852✔





700



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







701



  }










702



  return code;




1,890,295,037✔





703



}










704












705



static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void** ppReader, int32_t capacity,




1,891,502,698✔





706



                                SSDataBlock* pResBlock, const char* idstr) {










707



  int32_t             code = TSDB_CODE_SUCCESS;




1,891,502,698✔





708



  int32_t             lino = 0;




1,891,502,698✔





709



  int8_t              level = 0;




1,891,502,698✔





710



  STsdbReader*        pReader = NULL;




1,891,653,961✔





711



  SBlockLoadSuppInfo* pSup = NULL;




1,891,653,961✔





712












713



  TSDB_CHECK_NULL(pVnode, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,891,653,961✔





714



  TSDB_CHECK_NULL(pCond, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,891,653,961✔





715



  TSDB_CHECK_NULL(ppReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,891,653,961✔





716












717



  *ppReader = NULL;




1,891,653,961✔





718



  pReader = (STsdbReader*)taosMemoryCalloc(1, sizeof(*pReader));




1,891,490,914✔





719



  TSDB_CHECK_NULL(pReader, code, lino, _end, terrno);




1,890,936,264✔





720












721



  initReaderStatus(&pReader->status);




1,890,936,264✔





722



  getTsdbByRetentions(pVnode, pCond, pVnode->config.tsdbCfg.retentions, idstr, &level, &pReader->pTsdb);




1,891,419,930✔





723












724



  pReader->info.suid = pCond->suid;




1,891,535,242✔





725



  pReader->info.order = pCond->order;




1,891,575,180✔





726



  pReader->info.verRange = getQueryVerRange(pVnode, pCond);




1,891,270,056✔





727



  code = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows, &pReader->info.window);




1,890,972,945✔





728



  TSDB_CHECK_CODE(code, lino, _end);




1,890,896,267✔





729












730



  if (idstr == NULL) {




1,890,896,267✔





731



    idstr = "";




116,424,614✔





732



  }










733



  pReader->idStr = taosStrdup(idstr);




1,890,896,267✔





734



  TSDB_CHECK_NULL(pReader->idStr, code, lino, _end, terrno);




1,890,964,557✔





735












736



  pReader->type = pCond->type;




1,891,038,711✔





737



  pReader->bFilesetDelimited = false;




1,890,679,116✔





738



  pReader->blockInfoBuf.numPerBucket = 1000;  // 1000 tables per bucket




1,891,003,058✔





739












740



  if (pCond->numOfCols <= 0) {




1,891,268,481✔





741



    tsdbError("vgId:%d, invalid column number %d in query cond, %s", TD_VID(pVnode), pCond->numOfCols, idstr);




×







742



    TSDB_CHECK_CONDITION(pCond->numOfCols > 0, code, lino, _end, TSDB_CODE_INVALID_PARA);




×







743



  }










744












745



  // allocate buffer in order to load data blocks from file










746



  pSup = &pReader->suppInfo;




1,890,222,049✔





747



  pSup->tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID;




1,890,980,005✔





748



  code = setColumnIdSlotList(pSup, pCond->colList, pCond->pSlotList, pCond->numOfCols);




1,890,895,021✔





749



  TSDB_CHECK_CODE(code, lino, _end);




1,891,152,988✔





750












751



  code = initResBlockInfo(&pReader->resBlockInfo, capacity, pResBlock, pCond, pSup);




1,891,152,988✔





752



  TSDB_CHECK_CODE(code, lino, _end);




1,889,661,624✔





753












754



  code = tBlockDataCreate(&pReader->status.fileBlockData);




1,889,661,624✔





755



  TSDB_CHECK_CODE(code, lino, _end);




1,888,509,012✔





756












757



  if (pReader->suppInfo.colId[0] != PRIMARYKEY_TIMESTAMP_COL_ID) {




1,888,509,012✔





758



    tsdbError("the first column isn't primary timestamp, %d, %s", pReader->suppInfo.colId[0], pReader->idStr);




×







759



    TSDB_CHECK_CONDITION(pReader->suppInfo.colId[0] == PRIMARYKEY_TIMESTAMP_COL_ID, code, lino, _end,




×







760



                         TSDB_CODE_INVALID_PARA);










761



  }










762












763



  pReader->status.pPrimaryTsCol = taosArrayGet(pReader->resBlockInfo.pResBlock->pDataBlock, pSup->slotId[0]);




1,889,486,871✔





764



  TSDB_CHECK_NULL(pReader->status.pPrimaryTsCol, code, lino, _end, terrno);




1,890,991,724✔





765












766



  int32_t type = pReader->status.pPrimaryTsCol->info.type;




1,889,181,790✔





767



  if (type != TSDB_DATA_TYPE_TIMESTAMP) {




1,890,911,210✔





768



    tsdbError("the first column isn't primary timestamp in result block, actual: %s, %s", tDataTypes[type].name,




×







769



              pReader->idStr);










770



    TSDB_CHECK_CONDITION(type == TSDB_DATA_TYPE_TIMESTAMP, code, lino, _end, TSDB_CODE_INVALID_PARA);




×







771



  }










772












773



  code = tsdbInitReaderLock(pReader);




1,890,911,210✔





774



  TSDB_CHECK_CODE(code, lino, _end);




1,890,667,478✔





775












776



  pSup->args = pReader->pTsdb->pVnode->pBse;




1,890,667,478✔





777












778



  code = tsem_init(&pReader->resumeAfterSuspend, 0, 0);




1,889,782,002✔





779



  TSDB_CHECK_CODE(code, lino, _end);




1,888,318,424✔





780












781



  *ppReader = pReader;




1,888,318,424✔





782



  pReader = NULL;




1,888,433,624✔





783












784



_end:




1,888,433,624✔





785



  if (code != TSDB_CODE_SUCCESS) {




1,888,813,854✔





786



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







787



  }










788



  if (pReader) {




1,890,515,730✔





789



    tsdbReaderClose2(pReader);




×







790



  }










791



  return code;




1,890,515,730✔





792



}










793












794



static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFileReader* pFileReader, SArray* pIndexList) {




2,147,483,647✔





795



  int32_t              code = TSDB_CODE_SUCCESS;




2,147,483,647✔





796



  int32_t              lino = 0;




2,147,483,647✔





797



  int32_t              st = 0;




2,147,483,647✔





798



  int32_t              et1 = 0;




2,147,483,647✔





799



  int32_t              et2 = 0;




2,147,483,647✔





800



  int32_t              numOfTables = 0;




2,147,483,647✔





801



  const TBrinBlkArray* pBlkArray = NULL;




2,147,483,647✔





802



  STableUidList*       pList = NULL;




2,147,483,647✔





803



  SBrinBlk*            pBrinBlk = NULL;




2,147,483,647✔





804












805



  if (pFileReader == NULL) {




2,147,483,647✔





806



    goto _end;




2,147,483,647✔





807



  }










808












809



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




166,877,443✔





810



  TSDB_CHECK_NULL(pIndexList, code, lino, _end, TSDB_CODE_INVALID_PARA);




166,877,443✔





811












812



  st = taosGetTimestampUs();




167,162,465✔





813



  numOfTables = tSimpleHashGetSize(pReader->status.pTableMap);




167,162,465✔





814












815



  code = tsdbDataFileReadBrinBlk(pFileReader, &pBlkArray);




167,153,594✔





816



  TSDB_CHECK_CODE(code, lino, _end);




167,098,742✔





817












818



#if 0










819



  LRUHandle* handle = NULL;










820












821



  int32_t    code = tsdbCacheGetBlockIdx(pFileReader->pTsdb->biCache, pFileReader, &handle);










822



  if (code != TSDB_CODE_SUCCESS || handle == NULL) {










823



    goto _end;










824



  }










825












826












827



  SArray* aBlockIdx = (SArray*)taosLRUCacheValue(pFileReader->pTsdb->biCache, handle);










828



  size_t  num = taosArrayGetSize(aBlockIdx);










829



  if (num == 0) {










830



    tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle);










831



    return TSDB_CODE_SUCCESS;










832



  }










833



#endif










834












835



  // todo binary search to the start position










836



  et1 = taosGetTimestampUs();




167,152,280✔





837












838



  pList = &pReader->status.uidList;




167,152,280✔





839












840



  int32_t i = 0;




167,125,397✔





841



  int32_t j = 0;




167,125,397✔





842



  while (i < TARRAY2_SIZE(pBlkArray)) {




312,201,674✔





843



    pBrinBlk = &pBlkArray->data[i];




148,473,387✔





844



    if (pBrinBlk->maxTbid.suid < pReader->info.suid) {




148,412,347✔





845



      i += 1;




147,570✔





846



      continue;




147,570✔





847



    }










848












849



    if (pBrinBlk->minTbid.suid > pReader->info.suid) {  // not include the queried table/super table, quit the loop




148,300,565✔





850



      break;




440,367✔





851



    }










852












853



    TSDB_CHECK_CONDITION(




147,893,289✔





854



        (pBrinBlk->minTbid.suid <= pReader->info.suid) && (pBrinBlk->maxTbid.suid >= pReader->info.suid), code, lino,










855



        _end, TSDB_CODE_INTERNAL_ERROR);










856












857



    if (pBrinBlk->maxTbid.suid == pReader->info.suid && pBrinBlk->maxTbid.uid < pList->tableUidList[j]) {




147,868,697✔





858



      i += 1;




83,482✔





859



      continue;




83,482✔





860



    }










861












862



    if (pBrinBlk->minTbid.suid == pReader->info.suid && pBrinBlk->minTbid.uid > pList->tableUidList[numOfTables - 1]) {




147,777,940✔





863



      break;




20,464✔





864



    }










865












866



    const void* p1 = taosArrayPush(pIndexList, pBrinBlk);




147,760,157✔





867



    TSDB_CHECK_NULL(p1, code, lino, _end, terrno);




147,760,157✔





868












869



    i += 1;




147,760,157✔





870



    if (pBrinBlk->maxTbid.suid == pReader->info.suid) {




147,760,157✔





871



      while (j < numOfTables && pList->tableUidList[j] < pBrinBlk->maxTbid.uid) {




329,855,387✔





872



        j++;




196,087,470✔





873



      }










874



      if (j >= numOfTables) {




133,760,741✔





875



        break;




2,908,478✔





876



      }










877



    }










878



  }










879












880



  et2 = taosGetTimestampUs();




167,159,248✔





881



  tsdbDebug("load block index for %d/%d tables completed, elapsed time:%.2f ms, set BrinBlk:%.2f ms, size:%.2f Kb %s",




167,159,248✔





882



            numOfTables, (int32_t)pBlkArray->size, (et1 - st) / 1000.0, (et2 - et1) / 1000.0,










883



            pBlkArray->size * sizeof(SBrinBlk) / 1024.0, pReader->idStr);










884












885



  pReader->cost.headFileLoadTime += (et1 - st) / 1000.0;




167,159,248✔





886












887



_end:




2,147,483,647✔





888



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





889



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







890



  }










891



  return code;




2,147,483,647✔





892



}










893












894



static int32_t loadFileBlockBrinInfo(STsdbReader* pReader, SArray* pIndexList, SBlockNumber* pBlockNum,




2,147,483,647✔





895



                                     SArray* pTableScanInfoList) {










896



  int32_t         code = TSDB_CODE_SUCCESS;




2,147,483,647✔





897



  int32_t         lino = 0;




2,147,483,647✔





898



  int64_t         st = 0;




2,147,483,647✔





899



  bool            asc = false;




2,147,483,647✔





900



  STimeWindow     w = {0};




2,147,483,647✔





901



  SBrinRecordIter iter = {0};




2,147,483,647✔





902



  int32_t         numOfTables = 0;




2,147,483,647✔





903



  SBrinRecord*    pRecord = NULL;




2,147,483,647✔





904



  int32_t         k = 0;




2,147,483,647✔





905



  size_t          sizeInDisk = 0;




2,147,483,647✔





906












907



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





908



  TSDB_CHECK_NULL(pBlockNum, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





909



  TSDB_CHECK_NULL(pTableScanInfoList, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





910












911



  st = taosGetTimestampUs();




2,147,483,647✔





912



  asc = ASCENDING_TRAVERSE(pReader->info.order);




2,147,483,647✔





913



  w = pReader->info.window;




2,147,483,647✔





914



  numOfTables = tSimpleHashGetSize(pReader->status.pTableMap);




2,147,483,647✔





915












916



  // clear info for the new file










917



  cleanupInfoForNextFileset(pReader->status.pTableMap);




2,147,483,647✔





918



  initBrinRecordIter(&iter, pReader->pFileReader, pIndexList);




2,147,483,647✔





919












920



  while (1) {




579,649,651✔





921



    code = getNextBrinRecord(&iter, &pRecord);




2,147,483,647✔





922



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





923












924



    if (pRecord == NULL) {




2,147,483,647✔





925



      break;




2,147,483,647✔





926



    }










927












928



    if (pRecord->suid > pReader->info.suid) {




595,419,011✔





929



      break;




13,993,623✔





930



    }










931












932



    uint64_t uid = pReader->status.uidList.tableUidList[k];




582,528,238✔





933



    if (pRecord->suid < pReader->info.suid) {




582,530,768✔





934



      continue;




6,794,987✔





935



    }










936












937



    if (uid < pRecord->uid) {  // forward the table uid index




575,749,981✔





938



      while (k < numOfTables && pReader->status.uidList.tableUidList[k] < pRecord->uid) {




398,562,350✔





939



        k += 1;




199,375,170✔





940



      }










941












942



      if (k >= numOfTables) {




199,182,085✔





943



        break;




2,909,832✔





944



      }










945












946



      uid = pReader->status.uidList.tableUidList[k];




196,272,253✔





947



    }










948












949



    if (pRecord->uid < uid) {




572,815,928✔





950



      continue;




10,835,964✔





951



    }










952












953



    TSDB_CHECK_CONDITION((pRecord->suid == pReader->info.suid) && (uid == pRecord->uid), code, lino, _end,




562,006,941✔





954



                         TSDB_CODE_INTERNAL_ERROR);










955












956



    STableBlockScanInfo* pScanInfo = NULL;




561,998,228✔





957



    code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr);




561,991,011✔





958



    TSDB_CHECK_CODE(code, lino, _end);




561,989,464✔





959












960



    // here we should find the first timestamp that is greater than the lastProcKey










961



    // the window is an open interval NOW.










962



    if (asc) {




561,989,464✔





963



      w.skey = pScanInfo->lastProcKey.ts;




558,026,244✔





964



    } else {










965



      w.ekey = pScanInfo->lastProcKey.ts;




3,963,220✔





966



    }










967












968



    // NOTE: specialized for open interval










969



    if (((w.skey < INT64_MAX) && ((w.skey + 1) > w.ekey)) || (w.skey == INT64_MAX)) {




562,003,724✔





970



      k += 1;




×







971



      if (k >= numOfTables) {




×







972



        break;




×







973



      } else {










974



        continue;




×







975



      }










976



    }










977












978



    // 1. time range check










979



    if (pRecord->firstKey.key.ts > w.ekey || pRecord->lastKey.key.ts < w.skey) {




562,003,724✔





980



      continue;




5,245,901✔





981



    }










982



    // The data block's time range must intersect with the query time range










983



    if (pRecord->firstKey.key.ts > pReader->info.window.ekey || pRecord->lastKey.key.ts < pReader->info.window.skey) {




556,701,521✔





984



      continue;




11,473✔





985



    }










986












987



    if (asc) {




556,738,102✔





988



      if (pkCompEx(&pRecord->lastKey.key, &pScanInfo->lastProcKey) <= 0) {




1,107,527,958✔





989



        continue;




×







990



      }










991



    } else {










992



      if (pkCompEx(&pRecord->firstKey.key, &pScanInfo->lastProcKey) >= 0) {




5,904,840✔





993



        continue;




×







994



      }










995



    }










996












997



    // 2. version range check, version range is an CLOSED interval










998



    if (pRecord->minVer > pReader->info.verRange.maxVer || pRecord->maxVer < pReader->info.verRange.minVer) {




556,694,696✔





999



      continue;




×







1000



    }










1001












1002



    if (pScanInfo->pBlockList == NULL) {




556,716,524✔





1003



      pScanInfo->pBlockList = taosArrayInit(4, sizeof(SFileDataBlockInfo));




341,198,252✔





1004



      TSDB_CHECK_NULL(pScanInfo->pBlockList, code, lino, _end, terrno);




341,200,995✔





1005



    }










1006












1007



    if (pScanInfo->pBlockIdxList == NULL) {




556,701,534✔





1008



      pScanInfo->pBlockIdxList = taosArrayInit(4, sizeof(STableDataBlockIdx));




35,696,622✔





1009



      TSDB_CHECK_NULL(pScanInfo->pBlockIdxList, code, lino, _end, terrno);




35,698,215✔





1010



    }










1011












1012



    SFileDataBlockInfo blockInfo = {.tbBlockIdx = TARRAY_SIZE(pScanInfo->pBlockList)};




556,732,618✔





1013



    code = recordToBlockInfo(&blockInfo, pRecord);




556,768,052✔





1014



    TSDB_CHECK_CODE(code, lino, _end);




556,661,584✔





1015



    sizeInDisk += blockInfo.blockSize;




556,661,584✔





1016












1017



    const void* p1 = taosArrayPush(pScanInfo->pBlockList, &blockInfo);




556,661,584✔





1018



    TSDB_CHECK_NULL(p1, code, lino, _end, terrno);




556,695,606✔





1019












1020



    // todo: refactor to record the fileset skey/ekey










1021



    if (pScanInfo->filesetWindow.skey > pRecord->firstKey.key.ts) {




556,695,606✔





1022



      pScanInfo->filesetWindow.skey = pRecord->firstKey.key.ts;




341,211,231✔





1023



    }










1024












1025



    if (pScanInfo->filesetWindow.ekey < pRecord->lastKey.key.ts) {




556,702,173✔





1026



      pScanInfo->filesetWindow.ekey = pRecord->lastKey.key.ts;




556,665,188✔





1027



    }










1028












1029



    pBlockNum->numOfBlocks += 1;




556,751,550✔





1030



    if (taosArrayGetSize(pTableScanInfoList) == 0) {




556,670,114✔





1031



      p1 = taosArrayPush(pTableScanInfoList, &pScanInfo);




145,920,660✔





1032



      TSDB_CHECK_NULL(p1, code, lino, _end, terrno);




145,920,660✔





1033



    } else {










1034



      STableBlockScanInfo** p = taosArrayGetLast(pTableScanInfoList);




410,854,172✔





1035



      TSDB_CHECK_NULL(p, code, lino, _end, TSDB_CODE_INVALID_PARA);




410,847,446✔





1036












1037



      if ((*p)->uid != uid) {




410,847,446✔





1038



        p1 = taosArrayPush(pTableScanInfoList, &pScanInfo);




195,324,602✔





1039



        TSDB_CHECK_NULL(p1, code, lino, _end, terrno);




195,324,602✔





1040



      }










1041



    }










1042



  }










1043












1044



  pBlockNum->numOfSttFiles = pReader->status.pCurrentFileset->lvlArr->size;




2,147,483,647✔





1045



  int32_t total = pBlockNum->numOfSttFiles + pBlockNum->numOfBlocks;




2,147,483,647✔





1046












1047



  double el = (taosGetTimestampUs() - st) / 1000.0;




2,147,483,647✔





1048



  tsdbDebug(




2,147,483,647✔





1049



      "load block of %d tables completed, blocks:%d in %d tables, stt-files:%d, block-info-size:%.2f Kb, elapsed "










1050



      "time:%.2f ms %s",










1051



      numOfTables, pBlockNum->numOfBlocks, (int32_t)taosArrayGetSize(pTableScanInfoList), pBlockNum->numOfSttFiles,










1052



      sizeInDisk / 1000.0, el, pReader->idStr);










1053












1054



  pReader->cost.numOfBlocks += total;




2,147,483,647✔





1055



  pReader->cost.headFileLoadTime += el;




2,147,483,647✔





1056












1057



_end:




2,147,483,647✔





1058



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





1059



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1060



  }










1061



  clearBrinBlockIter(&iter);




2,147,483,647✔





1062



  return code;




2,147,483,647✔





1063



}










1064












1065



static void setBlockAllDumped(SFileBlockDumpInfo* pDumpInfo, int64_t maxKey, int32_t order) {




976,263,954✔





1066



  if (pDumpInfo != NULL) {




976,263,954✔





1067



    pDumpInfo->allDumped = true;




976,382,343✔





1068



  }










1069



}




976,360,829✔





1070












1071



static int32_t updateLastKeyInfo(SRowKey* pKey, SFileDataBlockInfo* pBlockInfo, SDataBlockInfo* pInfo, int32_t numOfPks,




532,718,856✔





1072



                                 bool asc) {










1073



  int32_t code = TSDB_CODE_SUCCESS;




532,718,856✔





1074



  int32_t lino = 0;




532,718,856✔





1075












1076



  TSDB_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA);




532,718,856✔





1077



  TSDB_CHECK_NULL(pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




532,718,856✔





1078












1079



  pKey->ts = asc ? pInfo->window.ekey : pInfo->window.skey;




532,718,856✔





1080



  pKey->numOfPKs = numOfPks;




532,756,077✔





1081



  if (pKey->numOfPKs <= 0) {




532,788,413✔





1082



    goto _end;




532,793,081✔





1083



  }










1084












1085



  TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




4,260✔





1086



  if (IS_NUMERIC_TYPE(pKey->pks[0].type)) {




4,260✔





1087



    VALUE_SET_TRIVIAL_DATUM(pKey->pks, asc ? pBlockInfo->lastPk.val : pBlockInfo->firstPk.val);




×







1088



  } else {










1089



    uint8_t* p = asc ? pBlockInfo->lastPk.pData : pBlockInfo->firstPk.pData;




4,260✔





1090



    pKey->pks[0].nData = asc ? varDataLen(pBlockInfo->lastPk.pData) : varDataLen(pBlockInfo->firstPk.pData);




4,260✔





1091



    TAOS_MEMCPY(pKey->pks[0].pData, p, pKey->pks[0].nData);




4,260✔





1092



  }










1093












1094



_end:




532,797,341✔





1095



  if (code != TSDB_CODE_SUCCESS) {




532,797,341✔





1096



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1097



  }










1098



  return code;




532,671,086✔





1099



}










1100












1101



static int32_t doGetValueFromBseBySeq(void* arg, uint8_t* pKey, int32_t keyLen, uint8_t** pValue, int32_t* len) {




123,620✔





1102



  int32_t  code = 0;




123,620✔





1103



  int32_t  lino = 0;




123,620✔





1104



  uint64_t seq = 0;




123,620✔





1105



  if (arg == NULL) {




123,620✔





1106



    tsdbError("failed to get value from bse by seq since %s", tstrerror(TSDB_CODE_INVALID_PARA));




×







1107



    return TSDB_CODE_INVALID_PARA;




×







1108



  }










1109












1110



  if (keyLen <= 0) {




123,620✔





1111



    *len = 0; 




×







1112



    return code;




×







1113



  } else {










1114



    int32_t unusedRet = tGetU64(pKey, &seq);




123,620✔





1115



  }










1116



 










1117



  if (seq == 0) {




123,620✔





1118



    tsdbError("failed to get value from bse by seq since seq is 0");




×







1119



    return TSDB_CODE_INVALID_PARA;




×







1120



  }










1121



  *len = 0;




123,620✔





1122



  code = bseGet((SBse*)arg, seq, pValue, len);




123,620✔





1123



  TSDB_CHECK_CODE(code, lino, _end);




123,620✔





1124












1125



_end:




123,620✔





1126



  if (code != 0) {




123,620✔





1127



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1128



  }










1129



  return code;




123,620✔





1130



}










1131



static int32_t doReallocBuf(SBlockLoadSuppInfo* pSup, int32_t colIndex, SColumnInfo* pInfo, int32_t len) {




158,940✔





1132



  int32_t code = 0;




158,940✔





1133



  int32_t bytes = pInfo->bytes;




158,940✔





1134



  int32_t extraSize = VARSTR_HEADER_SIZE;




158,940✔





1135












1136



  if (IS_STR_DATA_BLOB(pInfo->type)) {




158,940✔





1137



    if (len + BLOBSTR_HEADER_SIZE > TSDB_MAX_BLOB_LEN) {




158,940✔





1138



      tsdbWarn("column cid:%d actual data len %d is bigger than schema len %d", pSup->colId[colIndex], len, bytes);




×







1139



      return TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER;




×







1140



    }










1141



    extraSize = BLOBSTR_HEADER_SIZE;




158,940✔





1142












1143



  } else {










1144



    if (len + VARSTR_HEADER_SIZE > pInfo->bytes) {




×







1145



      tsdbWarn("column cid:%d actual data len %d is bigger than schema len %d", pSup->colId[colIndex], len, bytes);




×







1146



      return TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER;




×







1147



    }










1148



  }










1149












1150



  char* p = taosMemoryRealloc(pSup->buildBuf[colIndex], len + extraSize);




158,940✔





1151



  if (p == NULL) {




158,940✔





1152



    return terrno;




×







1153



  }










1154












1155



  pSup->buildBuf[colIndex] = p;




158,940✔





1156












1157



  if (IS_STR_DATA_BLOB(pInfo->type)) {




158,940✔





1158



    blobDataSetLen(pSup->buildBuf[colIndex], len);




158,940✔





1159



  } else {










1160



    varDataSetLen(pSup->buildBuf[colIndex], len);




×







1161



  }










1162












1163



  return code;




158,940✔





1164



}










1165



static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_t colIndex, SColVal* pColVal,




2,147,483,647✔





1166



                            SBlockLoadSuppInfo* pSup) {










1167



  int32_t code = TSDB_CODE_SUCCESS;




2,147,483,647✔





1168



int32_t lino = 0;




2,147,483,647✔





1169












1170



  uint8_t* pValue = 0;




2,147,483,647✔





1171



  int32_t  len = 0;




2,147,483,647✔





1172



  TSDB_CHECK_NULL(pColInfoData, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





1173



  TSDB_CHECK_NULL(pColVal, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





1174












1175



  if (IS_VAR_DATA_TYPE(pColVal->value.type)) {




2,147,483,647✔





1176



    if (!COL_VAL_IS_VALUE(pColVal)) {




2,147,483,647✔





1177



      colDataSetNULL(pColInfoData, rowIndex);




2,147,483,647✔





1178



    } else {










1179



      if (IS_STR_DATA_BLOB(pColVal->value.type)) {




2,147,483,647✔





1180



        if (pColVal->value.nData != 0) {




416,785,570✔





1181



          code = doGetValueFromBseBySeq(pSup->args, pColVal->value.pData, pColVal->value.nData, &pValue, &len);




123,620✔





1182



          if (code != 0) {




123,620✔





1183



            colDataSetNULL(pColInfoData, rowIndex);




×







1184



            return 0;




×







1185



          } else {










1186



            TSDB_CHECK_CODE(code, lino, _end);




123,620✔





1187



          }










1188



        }










1189












1190



        if (IS_STR_DATA_BLOB(pColVal->value.type)) {




158,940✔





1191



          code = doReallocBuf(pSup, colIndex, &pColInfoData->info, len);




158,940✔





1192



        } else {










1193



          varDataSetLen(pSup->buildBuf[colIndex], len);




×







1194



          if ((len + VARSTR_HEADER_SIZE) > pColInfoData->info.bytes) {




×







1195



            tsdbWarn("column cid:%d actual data len %d is bigger than schema len %d", pColVal->cid, len,




×







1196



                     pColInfoData->info.bytes);










1197



            code = TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER;




×







1198



            TSDB_CHECK_CODE(code, lino, _end);




×







1199



          }










1200



        }










1201












1202



        if (len > 0) {




158,940✔





1203



          if (IS_STR_DATA_BLOB(pColVal->value.type)) {




123,620✔





1204



            (void)memcpy(blobDataVal(pSup->buildBuf[colIndex]), pValue, len);




123,620✔





1205



          } else {










1206



            (void)memcpy(varDataVal(pSup->buildBuf[colIndex]), pValue, len);




×







1207



          }










1208



        }










1209



        code = colDataSetVal(pColInfoData, rowIndex, pSup->buildBuf[colIndex], false);




158,940✔





1210



        taosMemFreeClear(pValue);




158,940✔





1211












1212



        TSDB_CHECK_CODE(code, lino, _end);




158,940✔





1213



      } else {










1214



        TSDB_CHECK_NULL(pSup, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





1215



        varDataSetLen(pSup->buildBuf[colIndex], pColVal->value.nData);




2,147,483,647✔





1216



        if ((pColVal->value.nData + VARSTR_HEADER_SIZE) > pColInfoData->info.bytes) {




2,147,483,647✔





1217



          tsdbWarn("column cid:%d actual data len %d is bigger than schema len %d", pColVal->cid, pColVal->value.nData,




×







1218



                   pColInfoData->info.bytes);










1219



          code = TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER;




×







1220



          TSDB_CHECK_CODE(code, lino, _end);




×







1221



        }










1222












1223



        if (pColVal->value.nData > 0) {  // pData may be null, if nData is 0




2,147,483,647✔





1224



          (void)memcpy(varDataVal(pSup->buildBuf[colIndex]), pColVal->value.pData, pColVal->value.nData);




2,147,483,647✔





1225



        }










1226












1227



        code = colDataSetVal(pColInfoData, rowIndex, pSup->buildBuf[colIndex], false);




2,147,483,647✔





1228



        TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





1229



      }










1230



    }










1231



  } else {










1232



    code = colDataSetVal(pColInfoData, rowIndex, VALUE_GET_DATUM(&pColVal->value, pColVal->value.type),




2,147,483,647✔





1233



                         !COL_VAL_IS_VALUE(pColVal));




2,147,483,647✔





1234



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





1235



  }










1236












1237



_end:




2,147,483,647✔





1238



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





1239



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1240



  }










1241



  return code;




2,147,483,647✔





1242



}










1243












1244



static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInfo** pInfo, const char* id) {




2,147,483,647✔





1245



  int32_t code = TSDB_CODE_SUCCESS;




2,147,483,647✔





1246



  int32_t lino = 0;




2,147,483,647✔





1247












1248



  TSDB_CHECK_NULL(pBlockIter, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





1249



  TSDB_CHECK_NULL(pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





1250



  TSDB_CHECK_NULL(pBlockIter->blockList, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





1251












1252



  *pInfo = NULL;




2,147,483,647✔





1253












1254



  size_t num = TARRAY_SIZE(pBlockIter->blockList);




2,147,483,647✔





1255



  if (num == 0) {




2,147,483,647✔





1256



    // Some callers would attempt to call this function. Filter out certain normal cases and return directly to avoid










1257



    // generating excessive unnecessary error logs.










1258



    TSDB_CHECK_CONDITION(num == pBlockIter->numOfBlocks, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





1259



    return TSDB_CODE_INVALID_PARA;




2,147,483,647✔





1260



  }










1261












1262



  *pInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index);




2,147,483,647✔





1263



  TSDB_CHECK_NULL(*pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





1264












1265



_end:




2,147,483,647✔





1266



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





1267



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1268



  }










1269



  return code;




2,147,483,647✔





1270



}










1271












1272



static int32_t doBinarySearchKey(const TSKEY* keyList, int num, int pos, TSKEY key, int order) {




1,537,620✔





1273



  // start end position










1274



  int s, e;










1275



  s = pos;




1,537,620✔





1276












1277



  // check










1278



  if (!(keyList != NULL && pos >= 0 && pos < num && num > 0)) {




1,537,620✔





1279



    return -1;




×







1280



  }










1281



  if (order == TSDB_ORDER_ASC) {




1,537,620✔





1282



    // find the first position which is smaller than the key










1283



    e = num - 1;




1,247,858✔





1284



    if (key < keyList[pos]) return -1;




1,247,858✔





1285



    while (1) {




13,231,435✔





1286



      // check can return










1287



      if (key >= keyList[e]) return e;




14,479,293✔





1288



      if (key <= keyList[s]) return s;




14,479,293✔





1289



      if (e - s <= 1) return s;




14,458,152✔





1290












1291



      // change start or end position










1292



      int mid = s + (e - s + 1) / 2;




14,242,827✔





1293



      if (keyList[mid] > key)




14,242,827✔





1294



        e = mid;




5,222,491✔





1295



      else if (keyList[mid] < key)




9,019,744✔





1296



        s = mid;




8,008,944✔





1297



      else










1298



        return mid;




1,010,800✔





1299



    }










1300



  } else {  // DESC










1301



    // find the first position which is bigger than the key










1302



    e = 0;




289,762✔





1303



    if (key > keyList[pos]) return -1;




289,762✔





1304



    while (1) {




2,890,136✔





1305



      // check can return










1306



      if (key <= keyList[e]) return e;




3,179,898✔





1307



      if (key >= keyList[s]) return s;




3,179,898✔





1308



      if (s - e <= 1) return s;




3,179,898✔





1309












1310



      // change start or end position










1311



      int mid = s - (s - e + 1) / 2;




2,896,038✔





1312



      if (keyList[mid] < key)




2,896,038✔





1313



        e = mid;




1,470,354✔





1314



      else if (keyList[mid] > key)




1,425,684✔





1315



        s = mid;




1,419,782✔





1316



      else










1317



        return mid;




5,902✔





1318



    }










1319



  }










1320



}










1321












1322



// handle the repeat ts cases.










1323



static int32_t findFirstPos(const int64_t* pTsList, int32_t num, int32_t startPos, bool asc) {




1,537,620✔





1324



  int32_t i = startPos;




1,537,620✔





1325



  int64_t startTs = pTsList[startPos];




1,537,620✔





1326



  if (asc) {




1,537,620✔





1327



    while (i < num && (pTsList[i] == startTs)) {




2,495,716✔





1328



      i++;




1,247,858✔





1329



    }










1330



    return i - 1;




1,247,858✔





1331



  } else {










1332



    while (i >= 0 && (pTsList[i] == startTs)) {




579,524✔





1333



      i--;




289,762✔





1334



    }










1335



    return i + 1;




289,762✔





1336



  }










1337



}










1338












1339



static int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData, SBrinRecord* pRecord, int32_t pos) {




424,871,998✔





1340



  // NOTE: reverse the order to find the end position in data block










1341



  int32_t code = TSDB_CODE_SUCCESS;




424,871,998✔





1342



  int32_t lino = 0;




424,871,998✔





1343



  int32_t endPos = -1;




424,871,998✔





1344



  bool    asc = false;




424,871,998✔





1345












1346



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




424,871,998✔





1347



  TSDB_CHECK_NULL(pRecord, code, lino, _end, TSDB_CODE_INVALID_PARA);




424,871,998✔





1348












1349



  asc = ASCENDING_TRAVERSE(pReader->info.order);




424,871,998✔





1350












1351



  if (asc && pReader->info.window.ekey >= pRecord->lastKey.key.ts) {




425,005,027✔





1352



    endPos = pRecord->numRow - 1;




423,370,090✔





1353



  } else if (!asc && pReader->info.window.skey <= pRecord->firstKey.key.ts) {




1,632,287✔





1354



    endPos = 0;




957,667✔





1355



  } else {










1356



    int64_t key = asc ? pReader->info.window.ekey : pReader->info.window.skey;




742,458✔





1357



    endPos = doBinarySearchKey(pBlockData->aTSKEY, pRecord->numRow, pos, key, pReader->info.order);




742,458✔





1358



    if (endPos == -1) {




742,458✔





1359



      goto _end;




×







1360



    }










1361












1362



    endPos = findFirstPos(pBlockData->aTSKEY, pRecord->numRow, endPos, asc);




742,458✔





1363



  }










1364












1365



  if ((pReader->info.verRange.maxVer >= pRecord->minVer && pReader->info.verRange.maxVer < pRecord->maxVer) ||




425,040,853✔





1366



      (pReader->info.verRange.minVer <= pRecord->maxVer && pReader->info.verRange.minVer > pRecord->minVer)) {




425,025,587✔





1367



    int32_t i = endPos;




×







1368












1369



    if (asc) {




×







1370



      for (; i >= 0; --i) {




×







1371



        if (pBlockData->aVersion[i] <= pReader->info.verRange.maxVer) {




×







1372



          break;




×







1373



        }










1374



      }










1375



    } else {










1376



      for (; i < pRecord->numRow; ++i) {




×







1377



        if (pBlockData->aVersion[i] >= pReader->info.verRange.minVer) {




×







1378



          break;




×







1379



        }










1380



      }










1381



    }










1382












1383



    endPos = i;




×







1384



  }










1385












1386



_end:




424,987,969✔





1387



  if (code != TSDB_CODE_SUCCESS) {




424,987,969✔





1388



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1389



  }










1390



  return endPos;




424,861,613✔





1391



}










1392












1393



static int32_t copyPrimaryTsCol(const SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData,




425,084,710✔





1394



                                int32_t dumpedRows, bool asc) {










1395



  int32_t code = TSDB_CODE_SUCCESS;




425,084,710✔





1396



  int32_t lino = 0;




425,084,710✔





1397












1398



  TSDB_CHECK_CONDITION((pBlockData != NULL) && (pBlockData->aTSKEY != NULL), code, lino, _end, TSDB_CODE_INVALID_PARA);




425,084,710✔





1399



  TSDB_CHECK_NULL(pDumpInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




425,068,227✔





1400



  TSDB_CHECK_NULL(pColData, code, lino, _end, TSDB_CODE_INVALID_PARA);




425,068,227✔





1401



  TSDB_CHECK_CONDITION(dumpedRows >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA);




425,068,227✔





1402












1403



  if (asc) {




425,068,227✔





1404



    TAOS_MEMCPY(pColData->pData, &pBlockData->aTSKEY[pDumpInfo->rowIndex], dumpedRows * sizeof(int64_t));




424,108,393✔





1405



  } else {










1406



    int32_t startIndex = pDumpInfo->rowIndex - dumpedRows + 1;




959,834✔





1407



    TAOS_MEMCPY(pColData->pData, &pBlockData->aTSKEY[startIndex], dumpedRows * sizeof(int64_t));




958,259✔





1408












1409



    // todo: opt perf by extract the loop










1410



    // reverse the array list










1411



    int32_t  mid = dumpedRows >> 1u;




958,259✔





1412



    int64_t* pts = (int64_t*)pColData->pData;




958,259✔





1413



    for (int32_t j = 0; j < mid; ++j) {




515,147,048✔





1414



      int64_t t = pts[j];




514,188,789✔





1415



      pts[j] = pts[dumpedRows - j - 1];




514,188,849✔





1416



      pts[dumpedRows - j - 1] = t;




514,213,637✔





1417



    }










1418



  }










1419












1420



_end:




958,259✔





1421



  if (code != TSDB_CODE_SUCCESS) {




425,031,962✔





1422



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1423



  }










1424



  return code;




424,939,644✔





1425



}










1426












1427



// a faster version of copy procedure.










1428



static int32_t copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData,




807,092,002✔





1429



                               int32_t dumpedRows, bool asc) {










1430



  int32_t  code = TSDB_CODE_SUCCESS;




807,092,002✔





1431



  int32_t  lino = 0;




807,092,002✔





1432



  uint8_t* p = NULL;




807,092,002✔





1433



  int32_t  step = asc ? 1 : -1;




807,092,002✔





1434












1435



  TSDB_CHECK_NULL(pData, code, lino, _end, TSDB_CODE_INVALID_PARA);




807,092,002✔





1436



  TSDB_CHECK_NULL(pDumpInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




807,092,002✔





1437



  TSDB_CHECK_NULL(pColData, code, lino, _end, TSDB_CODE_INVALID_PARA);




807,092,002✔





1438



  TSDB_CHECK_CONDITION(dumpedRows >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA);




807,092,002✔





1439












1440



  TSDB_CHECK_CONDITION(pData->type < TSDB_DATA_TYPE_MAX, code, lino, _end, TSDB_CODE_INVALID_PARA);




807,092,002✔





1441



  if (asc) {




807,174,613✔





1442



    p = pData->pData + tDataTypes[pData->type].bytes * pDumpInfo->rowIndex;




807,163,639✔





1443



  } else {










1444



    int32_t startIndex = pDumpInfo->rowIndex - dumpedRows + 1;




10,974✔





1445



    p = pData->pData + tDataTypes[pData->type].bytes * startIndex;




10,974✔





1446



  }










1447












1448



  // make sure it is aligned to 8bit, the allocated memory address is aligned to 256bit










1449












1450



  // 1. copy data in a batch model










1451



  TAOS_MEMCPY(pColData->pData, p, dumpedRows * tDataTypes[pData->type].bytes);




807,272,729✔





1452












1453



  // 2. reverse the array list in case of descending order scan data block










1454



  if (!asc) {




807,224,620✔





1455



    switch (pColData->info.type) {




10,974✔





1456



      case TSDB_DATA_TYPE_TIMESTAMP:




×







1457



      case TSDB_DATA_TYPE_DOUBLE:










1458



      case TSDB_DATA_TYPE_BIGINT:










1459



      case TSDB_DATA_TYPE_DECIMAL64:










1460



      case TSDB_DATA_TYPE_UBIGINT: {










1461



        int32_t  mid = dumpedRows >> 1u;




×







1462



        int64_t* pts = (int64_t*)pColData->pData;




×







1463



        for (int32_t j = 0; j < mid; ++j) {




×







1464



          int64_t t = pts[j];




×







1465



          pts[j] = pts[dumpedRows - j - 1];




×







1466



          pts[dumpedRows - j - 1] = t;




×







1467



        }










1468



        break;




×







1469



      }










1470












1471



      case TSDB_DATA_TYPE_BOOL:




×







1472



      case TSDB_DATA_TYPE_TINYINT:










1473



      case TSDB_DATA_TYPE_UTINYINT: {










1474



        int32_t mid = dumpedRows >> 1u;




×







1475



        int8_t* pts = (int8_t*)pColData->pData;




×







1476



        for (int32_t j = 0; j < mid; ++j) {




×







1477



          int8_t t = pts[j];




×







1478



          pts[j] = pts[dumpedRows - j - 1];




×







1479



          pts[dumpedRows - j - 1] = t;




×







1480



        }










1481



        break;




×







1482



      }










1483












1484



      case TSDB_DATA_TYPE_SMALLINT:




×







1485



      case TSDB_DATA_TYPE_USMALLINT: {










1486



        int32_t  mid = dumpedRows >> 1u;




×







1487



        int16_t* pts = (int16_t*)pColData->pData;




×







1488



        for (int32_t j = 0; j < mid; ++j) {




×







1489



          int64_t t = pts[j];




×







1490



          pts[j] = pts[dumpedRows - j - 1];




×







1491



          pts[dumpedRows - j - 1] = t;




×







1492



        }










1493



        break;




×







1494



      }










1495












1496



      case TSDB_DATA_TYPE_FLOAT:




10,974✔





1497



      case TSDB_DATA_TYPE_INT:










1498



      case TSDB_DATA_TYPE_UINT: {










1499



        int32_t  mid = dumpedRows >> 1u;




10,974✔





1500



        int32_t* pts = (int32_t*)pColData->pData;




10,974✔





1501



        for (int32_t j = 0; j < mid; ++j) {




10,249,716✔





1502



          int32_t t = pts[j];




10,238,742✔





1503



          pts[j] = pts[dumpedRows - j - 1];




10,238,742✔





1504



          pts[dumpedRows - j - 1] = t;




10,238,742✔





1505



        }










1506



        break;




10,974✔





1507



      }










1508



      case TSDB_DATA_TYPE_DECIMAL: {




×







1509



        int32_t      mid = dumpedRows >> 1u;




×







1510



        DecimalWord* pDec = (DecimalWord*)pColData->pData;




×







1511



        DecimalWord  tmp[2] = {0};




×







1512



        for (int32_t j = 0; j < mid; ++j) {




×







1513



          tmp[0] = pDec[2 * j];




×







1514



          tmp[1] = pDec[2 * j + 1];




×







1515



          pDec[2 * j] = pDec[2 * (dumpedRows - j - 1)];




×







1516



          pDec[2 * j + 1] = pDec[2 * (dumpedRows - j - 1) + 1];




×







1517



          pDec[2 * (dumpedRows - j - 1)] = tmp[0];




×







1518



          pDec[2 * (dumpedRows - j - 1) + 1] = tmp[1];




×







1519



        }










1520



        break;




×







1521



      }










1522



    }










1523



  }










1524












1525



  // 3. if the  null value exists, check items one-by-one










1526



  if (pData->flag != HAS_VALUE) {




807,224,620✔





1527



    int32_t rowIndex = 0;




140,131,941✔





1528












1529



    for (int32_t j = pDumpInfo->rowIndex; rowIndex < dumpedRows; j += step, rowIndex++) {




2,147,483,647✔





1530



      uint8_t v = tColDataGetBitValue(pData, j);




2,147,483,647✔





1531



      if (v == 0 || v == 1) {




2,147,483,647✔





1532



        colDataSetNull_f(pColData->nullbitmap, rowIndex);




2,147,483,647✔





1533



        pColData->hasNull = true;




2,147,483,647✔





1534



      }










1535



    }










1536



  }










1537












1538



_end:




807,241,729✔





1539



  if (code != TSDB_CODE_SUCCESS) {




807,241,729✔





1540



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1541



  }










1542



  return code;




807,236,045✔





1543



}










1544












1545



static void blockInfoToRecord(SBrinRecord* record, SFileDataBlockInfo* pBlockInfo, SBlockLoadSuppInfo* pSupp) {




1,852,074,124✔





1546



  record->uid = pBlockInfo->uid;




1,852,074,124✔





1547



  record->firstKey = (STsdbRowKey){.key = {.ts = pBlockInfo->firstKey, .numOfPKs = pSupp->numOfPks}};




1,852,451,430✔





1548



  record->lastKey = (STsdbRowKey){.key = {.ts = pBlockInfo->lastKey, .numOfPKs = pSupp->numOfPks}};




1,852,473,902✔





1549












1550



  if (pSupp->numOfPks > 0) {




1,852,585,127✔





1551



    SValue* pFirst = &record->firstKey.key.pks[0];




29,820✔





1552



    SValue* pLast = &record->lastKey.key.pks[0];




29,820✔





1553












1554



    pFirst->type = pSupp->pk.type;




29,820✔





1555



    pLast->type = pSupp->pk.type;




29,820✔





1556












1557



    if (IS_VAR_DATA_TYPE(pFirst->type)) {




29,820✔





1558



      if (IS_STR_DATA_BLOB(pFirst->type)) {




29,820✔





1559



        pFirst->pData = (uint8_t*)blobDataVal(pBlockInfo->firstPk.pData);




×







1560



        pFirst->nData = blobDataLen(pBlockInfo->firstPk.pData);




×







1561












1562



        pLast->pData = (uint8_t*)blobDataVal(pBlockInfo->lastPk.pData);




×







1563



        pLast->nData = blobDataLen(pBlockInfo->lastPk.pData);




×







1564












1565



      } else {










1566



        pFirst->pData = (uint8_t*)varDataVal(pBlockInfo->firstPk.pData);




29,820✔





1567



        pFirst->nData = varDataLen(pBlockInfo->firstPk.pData);




29,820✔





1568












1569



        pLast->pData = (uint8_t*)varDataVal(pBlockInfo->lastPk.pData);




29,820✔





1570



        pLast->nData = varDataLen(pBlockInfo->lastPk.pData);




29,820✔





1571



      }










1572



    } else {










1573



      VALUE_SET_TRIVIAL_DATUM(pFirst, pBlockInfo->firstPk.val);




×







1574



      VALUE_SET_TRIVIAL_DATUM(pLast, pBlockInfo->lastPk.val);




×







1575



    }










1576



  }










1577












1578



  record->minVer = pBlockInfo->minVer;




1,852,497,141✔





1579



  record->maxVer = pBlockInfo->maxVer;




1,852,550,607✔





1580



  record->blockOffset = pBlockInfo->blockOffset;




1,852,551,837✔





1581



  record->smaOffset = pBlockInfo->smaOffset;




1,852,573,225✔





1582



  record->blockSize = pBlockInfo->blockSize;




1,852,533,528✔





1583



  record->blockKeySize = pBlockInfo->blockKeySize;




1,852,569,835✔





1584



  record->smaSize = pBlockInfo->smaSize;




1,852,528,465✔





1585



  record->numRow = pBlockInfo->numRow;




1,852,530,161✔





1586



  record->count = pBlockInfo->count;




1,852,527,893✔





1587



}




1,852,554,713✔





1588












1589



static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastProcKey) {




424,930,080✔





1590



  int32_t             code = TSDB_CODE_SUCCESS;




424,930,080✔





1591



  int32_t             lino = 0;




424,930,080✔





1592



  SReaderStatus*      pStatus = NULL;




424,930,080✔





1593



  SDataBlockIter*     pBlockIter = NULL;




424,930,080✔





1594



  SBlockLoadSuppInfo* pSupInfo = NULL;




424,930,080✔





1595



  SFileBlockDumpInfo* pDumpInfo = NULL;




424,930,080✔





1596



  SBlockData*         pBlockData = NULL;




424,930,080✔





1597



  SFileDataBlockInfo* pBlockInfo = NULL;




424,930,080✔





1598



  SSDataBlock*        pResBlock = NULL;




424,960,705✔





1599



  int32_t             numOfOutputCols = 0;




424,960,705✔





1600



  int64_t             st = 0;




424,960,705✔





1601



  bool                asc = false;




424,960,705✔





1602



  int32_t             step = 0;




424,960,705✔





1603



  SColVal             cv = {0};




424,960,705✔





1604



  SBrinRecord         tmp;




424,990,369✔





1605



  SBrinRecord*        pRecord = NULL;




424,990,394✔





1606












1607



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




424,990,394✔





1608












1609



  pStatus = &pReader->status;




424,990,394✔





1610



  pBlockIter = &pStatus->blockIter;




424,999,711✔





1611



  pSupInfo = &pReader->suppInfo;




424,928,718✔





1612












1613



  pSupInfo->args = pReader->pTsdb->pVnode->pBse;




425,090,343✔





1614



  pDumpInfo = &pReader->status.fBlockDumpInfo;




425,104,227✔





1615












1616



  pBlockData = &pStatus->fileBlockData;




425,102,225✔





1617



  pResBlock = pReader->resBlockInfo.pResBlock;




425,105,906✔





1618



  numOfOutputCols = pSupInfo->numOfCols;




425,093,520✔





1619



  st = taosGetTimestampUs();




425,109,118✔





1620



  asc = ASCENDING_TRAVERSE(pReader->info.order);




425,109,118✔





1621



  step = asc ? 1 : -1;




425,103,377✔





1622












1623



  code = getCurrentBlockInfo(pBlockIter, &pBlockInfo, pReader->idStr);




425,103,377✔





1624



  TSDB_CHECK_CODE(code, lino, _end);




425,045,236✔





1625












1626



  blockInfoToRecord(&tmp, pBlockInfo, pSupInfo);




425,045,236✔





1627



  pRecord = &tmp;




425,091,272✔





1628












1629



  // no data exists, return directly.










1630



  if (pBlockData->nRow == 0 || pBlockData->aTSKEY == 0) {




425,091,272✔





1631



    tsdbWarn("%p no need to copy since no data in blockData, table uid:%" PRIu64 " has been dropped, %s", pReader,




65,379✔





1632



             pBlockInfo->uid, pReader->idStr);










1633



    pResBlock->info.rows = 0;




65,379✔





1634



    goto _end;




×







1635



  }










1636












1637



  TSDB_CHECK_CONDITION((pDumpInfo->rowIndex >= 0) && (pDumpInfo->rowIndex < pRecord->numRow), code, lino, _end,




425,007,758✔





1638



                       TSDB_CODE_INVALID_PARA);










1639












1640



  // row index of dump info remain the initial position, let's find the appropriate start position.










1641



  if (((pDumpInfo->rowIndex == 0) && asc) || ((pDumpInfo->rowIndex == (pRecord->numRow - 1)) && (!asc))) {




424,980,036✔





1642



    if (asc && pReader->info.window.skey <= pRecord->firstKey.key.ts &&




424,841,908✔





1643



        pReader->info.verRange.minVer <= pRecord->minVer) {




423,803,695✔





1644



      // pDumpInfo->rowIndex = 0;










1645



    } else if (!asc && pReader->info.window.ekey >= pRecord->lastKey.key.ts &&




1,247,709✔





1646



               pReader->info.verRange.maxVer >= pRecord->maxVer) {




452,547✔





1647



      // pDumpInfo->rowIndex = pRecord->numRow - 1;










1648



    } else {  // find the appropriate the start position in current block, and set it to be the current rowIndex










1649



      int32_t pos = asc ? pRecord->numRow - 1 : 0;




795,162✔





1650



      int32_t order = asc ? TSDB_ORDER_DESC : TSDB_ORDER_ASC;




795,162✔





1651



      int64_t key = asc ? pReader->info.window.skey : pReader->info.window.ekey;




795,162✔





1652



      pDumpInfo->rowIndex = doBinarySearchKey(pBlockData->aTSKEY, pRecord->numRow, pos, key, order);




795,162✔





1653












1654



      if (pDumpInfo->rowIndex < 0) {




795,162✔





1655



        tsdbError(




×







1656



            "%p failed to locate the start position in current block, global index:%d, table index:%d, brange:%" PRId64










1657



            "-%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64 " %s",










1658



            pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pRecord->firstKey.key.ts, pRecord->lastKey.key.ts,










1659



            pRecord->minVer, pRecord->maxVer, pReader->idStr);










1660



        code = TSDB_CODE_INVALID_PARA;




×







1661



        TSDB_CHECK_CODE(code, lino, _end);




×







1662



      }










1663












1664



      pDumpInfo->rowIndex = findFirstPos(pBlockData->aTSKEY, pRecord->numRow, pDumpInfo->rowIndex, (!asc));




795,162✔





1665












1666



      TSDB_CHECK_CONDITION(




795,162✔





1667



          (pReader->info.verRange.minVer <= pRecord->maxVer && pReader->info.verRange.maxVer >= pRecord->minVer), code,










1668



          lino, _end, TSDB_CODE_INVALID_PARA);










1669












1670



      // find the appropriate start position that satisfies the version requirement.










1671



      if ((pReader->info.verRange.maxVer >= pRecord->minVer && pReader->info.verRange.maxVer < pRecord->maxVer) ||




795,162✔





1672



          (pReader->info.verRange.minVer <= pRecord->maxVer && pReader->info.verRange.minVer > pRecord->minVer)) {




795,162✔





1673



        int32_t i = pDumpInfo->rowIndex;




×







1674



        if (asc) {




×







1675



          for (; i < pRecord->numRow; ++i) {




×







1676



            if (pBlockData->aVersion[i] >= pReader->info.verRange.minVer) {




×







1677



              break;




×







1678



            }










1679



          }










1680



        } else {










1681



          for (; i >= 0; --i) {




×







1682



            if (pBlockData->aVersion[i] <= pReader->info.verRange.maxVer) {




×







1683



              break;




×







1684



            }










1685



          }










1686



        }










1687












1688



        pDumpInfo->rowIndex = i;




×







1689



      }










1690



    }










1691



  }










1692












1693



  // time window check










1694



  int32_t endIndex = getEndPosInDataBlock(pReader, pBlockData, pRecord, pDumpInfo->rowIndex);




425,236,452✔





1695



  if (endIndex == -1) {




424,852,443✔





1696



    setBlockAllDumped(pDumpInfo, pReader->info.window.ekey, pReader->info.order);




×







1697



    goto _end;




×







1698



  }










1699












1700



  endIndex += step;




424,852,443✔





1701



  int32_t dumpedRows = asc ? (endIndex - pDumpInfo->rowIndex) : (pDumpInfo->rowIndex - endIndex);




424,852,443✔





1702



  if (dumpedRows > pReader->resBlockInfo.capacity) {  // output buffer check




424,923,985✔





1703



    dumpedRows = pReader->resBlockInfo.capacity;




×







1704



  } else if (dumpedRows <= 0) {  // no qualified rows in current data block, quit directly.




424,893,711✔





1705



    setBlockAllDumped(pDumpInfo, pReader->info.window.ekey, pReader->info.order);




×







1706



    goto _end;




×







1707



  }










1708












1709



  int32_t i = 0;




424,893,711✔





1710



  int32_t rowIndex = 0;




424,893,711✔





1711












1712



  SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);




424,893,711✔





1713



  TSDB_CHECK_NULL(pColData, code, lino, _end, TSDB_CODE_INVALID_PARA);




425,117,376✔





1714












1715



  if (pSupInfo->colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) {




425,117,376✔





1716



    code = copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc);




425,106,332✔





1717



    TSDB_CHECK_CODE(code, lino, _end);




424,970,748✔





1718



    i += 1;




424,970,748✔





1719



  }










1720












1721



  int32_t colIndex = 0;




424,973,515✔





1722



  int32_t num = pBlockData->nColData;




424,973,515✔





1723



  while (i < numOfOutputCols && colIndex < num) {




1,540,982,419✔





1724



    rowIndex = 0;




1,115,929,024✔





1725












1726



    SColData* pData = tBlockDataGetColDataByIdx(pBlockData, colIndex);




1,115,929,024✔





1727



    if (pData->cid < pSupInfo->colId[i]) {




1,115,963,265✔





1728



      colIndex += 1;




×







1729



    } else if (pData->cid == pSupInfo->colId[i]) {




1,116,036,559✔





1730



      pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);




1,116,061,323✔





1731



      TSDB_CHECK_NULL(pColData, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,116,118,873✔





1732












1733



      if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) {




1,116,118,873✔





1734



        colDataSetNNULL(pColData, 0, dumpedRows);




88,081,406✔





1735



      } else {










1736



        if (IS_MATHABLE_TYPE(pColData->info.type)) {




1,028,072,804✔





1737



          code = copyNumericCols(pData, pDumpInfo, pColData, dumpedRows, asc);




807,165,916✔





1738



          TSDB_CHECK_CODE(code, lino, _end);




807,245,874✔





1739



        } else {  // varchar/nchar type










1740



          for (int32_t j = pDumpInfo->rowIndex; rowIndex < dumpedRows; j += step) {




2,147,483,647✔





1741



            code = tColDataGetValue(pData, j, &cv);




2,147,483,647✔





1742



            TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





1743



            code = doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo);




2,147,483,647✔





1744



            TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





1745



          }










1746



        }










1747



      }










1748












1749



      colIndex += 1;




1,115,969,778✔





1750



      i += 1;




1,115,969,778✔





1751



    } else {  // the specified column does not exist in file block, fill with null data










1752



      pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);




×







1753



      TSDB_CHECK_NULL(pColData, code, lino, _end, TSDB_CODE_INVALID_PARA);




×







1754












1755



      colDataSetNNULL(pColData, 0, dumpedRows);




×







1756



      i += 1;




×







1757



    }










1758



  }










1759












1760



  // fill the mis-matched columns with null value










1761



  while (i < numOfOutputCols) {




425,053,395✔





1762



    pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);




×







1763



    TSDB_CHECK_NULL(pColData, code, lino, _end, TSDB_CODE_INVALID_PARA);




×







1764












1765



    colDataSetNNULL(pColData, 0, dumpedRows);




×







1766



    i += 1;




×







1767



  }










1768












1769



  pResBlock->info.dataLoad = 1;




425,053,395✔





1770



  pResBlock->info.rows = dumpedRows;




425,047,463✔





1771



  pDumpInfo->rowIndex += step * dumpedRows;




425,052,203✔





1772












1773



  tColRowGetKeyDeepCopy(pBlockData, pDumpInfo->rowIndex - step, pSupInfo->pkSrcSlot, pLastProcKey);




425,013,994✔





1774












1775



  // check if current block are all handled










1776



  if (pDumpInfo->rowIndex >= 0 && pDumpInfo->rowIndex < pRecord->numRow) {




425,838,383✔





1777



    int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex];




742,458✔





1778












1779



    // the remain data has out of query time window, ignore current block










1780



    if (outOfTimeWindow(ts, &pReader->info.window)) {




742,458✔





1781



      setBlockAllDumped(pDumpInfo, ts, pReader->info.order);




742,458✔





1782



    }










1783



  } else {










1784



    int64_t ts = asc ? pRecord->lastKey.key.ts : pRecord->firstKey.key.ts;




424,224,002✔





1785



    setBlockAllDumped(pDumpInfo, ts, pReader->info.order);




424,303,320✔





1786



  }










1787












1788



  double elapsedTime = (taosGetTimestampUs() - st) / 1000.0;




425,065,096✔





1789



  pReader->cost.blockLoadTime += elapsedTime;




425,065,096✔





1790












1791



  int32_t unDumpedRows = asc ? pRecord->numRow - pDumpInfo->rowIndex : pDumpInfo->rowIndex + 1;




424,992,381✔





1792



  tsdbDebug("%p copy file block to sdatablock, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64




424,926,837✔





1793



            ", rows:%d, remain:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", uid:%" PRIu64 " elapsed time:%.2f ms, %s",










1794



            pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pRecord->firstKey.key.ts, pRecord->lastKey.key.ts,










1795



            dumpedRows, unDumpedRows, pRecord->minVer, pRecord->maxVer, pBlockInfo->uid, elapsedTime, pReader->idStr);










1796












1797



_end:




424,926,837✔





1798



  if (code != TSDB_CODE_SUCCESS) {




425,003,265✔





1799



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1800



  }










1801



  return code;




425,003,265✔





1802



}










1803












1804



static FORCE_INLINE STSchema* getTableSchemaImpl(STsdbReader* pReader, uint64_t uid) {










1805



  int32_t code = TSDB_CODE_SUCCESS;




326,420✔





1806



  int32_t lino = 0;




326,420✔





1807












1808



  TSDB_CHECK_CONDITION((pReader != NULL) && (pReader->info.pSchema == NULL), code, lino, _end, TSDB_CODE_INVALID_PARA);




326,420✔





1809












1810



  code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, uid, -1, &pReader->info.pSchema);




326,420✔





1811



  if (code != TSDB_CODE_SUCCESS || pReader->info.pSchema == NULL) {




326,420✔





1812



    terrno = code;




×







1813



    tsdbError("failed to get table schema, suid:%"PRIu64",uid:%" PRIu64 ", it may have been dropped, ver:-1, %s", pReader->info.suid, uid, pReader->idStr);




×







1814



  }










1815



  TSDB_CHECK_CODE(code, lino, _end);




326,420✔





1816



  TSDB_CHECK_NULL(pReader->info.pSchema, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);




326,420✔





1817












1818



  code = tsdbRowMergerInit(&pReader->status.merger, pReader->info.pSchema);




326,420✔





1819



  TSDB_CHECK_CODE(code, lino, _end);




326,420✔





1820












1821



_end:




326,420✔





1822



  if (code != TSDB_CODE_SUCCESS) {




326,420✔





1823



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1824



    terrno = code;




×







1825



    return NULL;




×







1826



  }










1827



  return pReader->info.pSchema;




326,420✔





1828



}










1829












1830



static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockIter, SBlockData* pBlockData,




440,074,128✔





1831



                                   uint64_t uid) {










1832



  int32_t             code = TSDB_CODE_SUCCESS;




440,074,128✔





1833



  int32_t             lino = 0;




440,074,128✔





1834



  STSchema*           pSchema = NULL;




440,074,128✔





1835



  SFileDataBlockInfo* pBlockInfo = NULL;




440,074,128✔





1836



  SBlockLoadSuppInfo* pSup = NULL;




440,145,323✔





1837



  SFileBlockDumpInfo* pDumpInfo = NULL;




440,145,323✔





1838



  int64_t             st = 0;




440,145,323✔





1839



  SBrinRecord         tmp;




440,145,323✔





1840



  SBrinRecord*        pRecord = NULL;




440,146,439✔





1841












1842



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




440,146,439✔





1843



  TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA);




440,146,439✔





1844












1845



  pSchema = pReader->info.pSchema;




440,146,439✔





1846



  st = taosGetTimestampUs();




440,201,390✔





1847



  pSup = &pReader->suppInfo;




440,201,390✔





1848












1849



  tBlockDataReset(pBlockData);




440,180,121✔





1850












1851



  if (pReader->info.pSchema == NULL) {




440,109,476✔





1852



    pSchema = getTableSchemaImpl(pReader, uid);




×







1853



    if (pSchema == NULL) {




×







1854



      tsdbError("%p table uid:%" PRIu64 " failed to get tableschema, code:%s, %s", pReader, uid, tstrerror(code),




×







1855



                pReader->idStr);










1856



      TSDB_CHECK_NULL(pSchema, code, lino, _end, terrno);




×







1857



    }










1858



  }










1859












1860



  code = getCurrentBlockInfo(pBlockIter, &pBlockInfo, pReader->idStr);




440,129,660✔





1861



  TSDB_CHECK_CODE(code, lino, _end);




440,194,733✔





1862












1863



  pDumpInfo = &pReader->status.fBlockDumpInfo;




440,194,733✔





1864












1865



  blockInfoToRecord(&tmp, pBlockInfo, pSup);




440,230,223✔





1866



  pRecord = &tmp;




440,255,574✔





1867



  code = tsdbDataFileReadBlockDataByColumn(pReader->pFileReader, pRecord, pBlockData, pSchema, &pSup->colId[1],




440,238,849✔





1868



                                           pSup->numOfCols - 1);




440,255,574✔





1869



  if (code != TSDB_CODE_SUCCESS) {




440,121,566✔





1870



    tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64




×







1871



              ", rows:%d, code:%s %s",










1872



              pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->firstKey, pBlockInfo->lastKey,










1873



              pBlockInfo->numRow, tstrerror(code), pReader->idStr);










1874



    TSDB_CHECK_CODE(code, lino, _end);




×







1875



  }










1876












1877



  double elapsedTime = (taosGetTimestampUs() - st) / 1000.0;




440,122,106✔





1878












1879



  tsdbDebug("%p load file block into buffer, global index:%d, index in table block list:%d, brange:%" PRId64 "-%" PRId64




440,122,106✔





1880



            ", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s",










1881



            pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pRecord->firstKey.key.ts, pRecord->lastKey.key.ts,










1882



            pRecord->numRow, pRecord->minVer, pRecord->maxVer, elapsedTime, pReader->idStr);










1883












1884



  pReader->cost.blockLoadTime += elapsedTime;




440,122,106✔





1885



  pDumpInfo->allDumped = false;




440,229,476✔





1886












1887



_end:




440,205,983✔





1888



  if (code != TSDB_CODE_SUCCESS) {




440,230,942✔





1889



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1890



  }










1891



  return code;




440,230,942✔





1892



}










1893












1894



/**










1895



 * This is an two rectangles overlap cases.










1896



 */










1897



static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* pVerRange, SFileDataBlockInfo* pBlock) {




566,808,237✔





1898



  return (pWindow->ekey < pBlock->lastKey && pWindow->ekey >= pBlock->firstKey) ||




569,995,315✔





1899



         (pWindow->skey > pBlock->firstKey && pWindow->skey <= pBlock->lastKey) ||




563,680,193✔





1900



         (pVerRange->minVer > pBlock->minVer && pVerRange->minVer <= pBlock->maxVer) ||




1,696,547,373✔





1901



         (pVerRange->maxVer < pBlock->maxVer && pVerRange->maxVer >= pBlock->minVer);




562,849,346✔





1902



}










1903












1904



static int32_t getNeighborBlockOfTable(SDataBlockIter* pBlockIter, SFileDataBlockInfo* pBlockInfo,




581,952,838✔





1905



                                       STableBlockScanInfo* pScanInfo, int32_t* nextIndex, int32_t order,










1906



                                       SBrinRecord* pRecord, SBlockLoadSuppInfo* pSupInfo, bool* res) {










1907



  int32_t             code = TSDB_CODE_SUCCESS;




581,952,838✔





1908



  int32_t             lino = 0;




581,952,838✔





1909



  bool                asc = false;




581,952,838✔





1910



  int32_t             step = 0;




581,952,838✔





1911



  STableDataBlockIdx* pTableDataBlockIdx = NULL;




581,952,838✔





1912



  SFileDataBlockInfo* p = NULL;




581,952,838✔





1913












1914



  TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




581,952,838✔





1915



  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




581,952,838✔





1916



  TSDB_CHECK_NULL(res, code, lino, _end, TSDB_CODE_INVALID_PARA);




581,952,838✔





1917












1918



  *res = false;




581,952,838✔





1919



  asc = ASCENDING_TRAVERSE(order);




581,916,584✔





1920



  step = asc ? 1 : -1;




581,916,584✔





1921












1922



  if (asc && pBlockInfo->tbBlockIdx >= taosArrayGetSize(pScanInfo->pBlockIdxList) - 1) {




581,916,584✔





1923



    *res = false;




348,215,692✔





1924



  } else if (!asc && pBlockInfo->tbBlockIdx == 0) {




233,737,509✔





1925



    *res = false;




2,807,117✔





1926



  } else {










1927



    TSDB_CHECK_NULL(pBlockIter, code, lino, _end, TSDB_CODE_INVALID_PARA);




231,042,442✔





1928



    TSDB_CHECK_NULL(nextIndex, code, lino, _end, TSDB_CODE_INVALID_PARA);




231,042,442✔





1929



    TSDB_CHECK_NULL(pRecord, code, lino, _end, TSDB_CODE_INVALID_PARA);




231,042,442✔





1930



    TSDB_CHECK_NULL(pSupInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




231,042,442✔





1931












1932



    pTableDataBlockIdx = taosArrayGet(pScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx + step);




231,042,442✔





1933



    TSDB_CHECK_NULL(pTableDataBlockIdx, code, lino, _end, TSDB_CODE_INVALID_PARA);




231,046,630✔





1934












1935



    p = taosArrayGet(pBlockIter->blockList, pTableDataBlockIdx->globalIndex);




231,046,630✔





1936



    TSDB_CHECK_NULL(p, code, lino, _end, TSDB_CODE_INVALID_PARA);




231,048,546✔





1937












1938



    blockInfoToRecord(pRecord, p, pSupInfo);




231,048,546✔





1939












1940



    *nextIndex = pBlockInfo->tbBlockIdx + step;




231,038,276✔





1941



    *res = true;




231,036,694✔





1942



  }










1943












1944



_end:




582,080,202✔





1945



  if (code != TSDB_CODE_SUCCESS) {




582,080,202✔





1946



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







1947



  }










1948



  return code;




581,984,697✔





1949



}










1950












1951



static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t index,




8,220✔





1952



                                             int32_t step) {










1953



  int32_t              code = TSDB_CODE_SUCCESS;




8,220✔





1954



  int32_t              lino = 0;




8,220✔





1955



  const void*          p = NULL;




8,220✔





1956



  SFileDataBlockInfo   fblock;




8,220✔





1957



  SFileDataBlockInfo*  pBlockInfo = NULL;




8,220✔





1958



  STableBlockScanInfo* pBlockScanInfo = NULL;




8,220✔





1959



  STableDataBlockIdx*  pTableDataBlockIdx = NULL;




8,220✔





1960












1961



  TSDB_CHECK_NULL(pBlockIter, code, lino, _end, TSDB_CODE_INVALID_PARA);




8,220✔





1962



  TSDB_CHECK_CONDITION((index >= 0) && (index < pBlockIter->numOfBlocks), code, lino, _end, TSDB_CODE_INVALID_PARA);




8,220✔





1963












1964



  p = taosArrayGet(pBlockIter->blockList, index);




8,220✔





1965



  TSDB_CHECK_NULL(p, code, lino, _end, TSDB_CODE_INVALID_PARA);




8,220✔





1966












1967



  fblock = *(SFileDataBlockInfo*)p;




8,220✔





1968



  pBlockIter->index += step;




8,220✔





1969












1970



  if (index != pBlockIter->index) {




8,220✔





1971



    if (index > pBlockIter->index) {




×







1972



      for (int32_t i = index - 1; i >= pBlockIter->index; --i) {




×







1973



        pBlockInfo = taosArrayGet(pBlockIter->blockList, i);




×







1974



        TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




×







1975












1976



        code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr);




×







1977



        TSDB_CHECK_CODE(code, lino, _end);




×







1978












1979



        pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx);




×







1980



        TSDB_CHECK_NULL(pTableDataBlockIdx, code, lino, _end, TSDB_CODE_INVALID_PARA);




×







1981












1982



        pTableDataBlockIdx->globalIndex = i + 1;




×







1983












1984



        taosArraySet(pBlockIter->blockList, i + 1, pBlockInfo);




×







1985



      }










1986



    } else if (index < pBlockIter->index) {




×







1987



      for (int32_t i = index + 1; i <= pBlockIter->index; ++i) {




×







1988



        pBlockInfo = taosArrayGet(pBlockIter->blockList, i);




×







1989



        TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




×







1990












1991



        code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr);




×







1992



        TSDB_CHECK_CODE(code, lino, _end);




×







1993












1994



        pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx);




×







1995



        TSDB_CHECK_NULL(pTableDataBlockIdx, code, lino, _end, TSDB_CODE_INVALID_PARA);




×







1996












1997



        pTableDataBlockIdx->globalIndex = i - 1;




×







1998



        taosArraySet(pBlockIter->blockList, i - 1, pBlockInfo);




×







1999



      }










2000



    }










2001












2002



    taosArraySet(pBlockIter->blockList, pBlockIter->index, &fblock);




×







2003



    pBlockScanInfo = NULL;




×







2004



    code = getTableBlockScanInfo(pReader->status.pTableMap, fblock.uid, &pBlockScanInfo, pReader->idStr);




×







2005



    TSDB_CHECK_CODE(code, lino, _end);




×







2006












2007



    pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, fblock.tbBlockIdx);




×







2008



    TSDB_CHECK_NULL(pTableDataBlockIdx, code, lino, _end, TSDB_CODE_INVALID_PARA);




×







2009












2010



    pTableDataBlockIdx->globalIndex = pBlockIter->index;




×







2011



  }










2012












2013



_end:




8,220✔





2014



  if (code != TSDB_CODE_SUCCESS) {




8,220✔





2015



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







2016



  }










2017



  return code;




8,220✔





2018



}










2019












2020



// todo: this attribute could be acquired during extractin the global ordered block list.










2021



static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* pRec, int32_t order, int32_t pkType,




231,019,116✔





2022



                                      int32_t numOfPk) {










2023



  // it is the last block in current file, no chance to overlap with neighbor blocks.










2024



  if (ASCENDING_TRAVERSE(order)) {




231,019,116✔





2025



    if (pBlock->lastKey == pRec->firstKey.key.ts) {




230,522,685✔





2026



      if (numOfPk > 0) {




12,330✔





2027



        SValue v1 = {.type = pkType};




×







2028



        if (IS_VAR_DATA_TYPE(pkType)) {




×







2029



          if (IS_STR_DATA_BLOB(pkType)) {




×







2030



            v1.pData = (uint8_t*)blobDataVal(pBlock->lastPk.pData), v1.nData = blobDataLen(pBlock->lastPk.pData);




×







2031



          } else {










2032



            v1.pData = (uint8_t*)varDataVal(pBlock->lastPk.pData), v1.nData = varDataLen(pBlock->lastPk.pData);




×







2033



          }










2034



        } else {










2035



          VALUE_SET_TRIVIAL_DATUM(&v1, pBlock->lastPk.val);




×







2036



        }










2037



        return (tValueCompare(&v1, &pRec->firstKey.key.pks[0]) == 0);




×







2038



      } else {  // no pk










2039



        return true;




12,330✔





2040



      }










2041



    } else {










2042



      return false;




230,506,523✔





2043



    }










2044



  } else {










2045



    if (pBlock->firstKey == pRec->lastKey.key.ts) {




496,431✔





2046



      if (numOfPk > 0) {




12,330✔





2047



        SValue v1 = {.type = pkType};




×







2048



        if (IS_VAR_DATA_TYPE(pkType)) {




×







2049



          if (IS_STR_DATA_BLOB(pkType)) {




×







2050



            v1.pData = (uint8_t*)blobDataVal(pBlock->firstPk.pData), v1.nData = blobDataLen(pBlock->firstPk.pData);




×







2051












2052



          } else {










2053



            v1.pData = (uint8_t*)varDataVal(pBlock->firstPk.pData), v1.nData = varDataLen(pBlock->firstPk.pData);




×







2054



          }










2055



        } else {










2056



          VALUE_SET_TRIVIAL_DATUM(&v1, pBlock->firstPk.val);




×







2057



        }










2058



        return (tValueCompare(&v1, &pRec->lastKey.key.pks[0]) == 0);




×







2059



      } else {  // no pk










2060



        return true;




12,330✔





2061



      }










2062



    } else {










2063



      return false;




486,017✔





2064



    }










2065



  }










2066



}










2067












2068



static int64_t getBoarderKeyInFiles(SFileDataBlockInfo* pBlock, STableBlockScanInfo* pScanInfo, int32_t order) {




533,529,835✔





2069



  bool ascScan = ASCENDING_TRAVERSE(order);




533,529,835✔





2070












2071



  int64_t key = 0;




533,529,835✔





2072



  if (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA) {




533,529,835✔





2073



    int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey.ts;




944,706✔





2074



    key = ascScan ? TMIN(pBlock->firstKey, keyInStt) : TMAX(pBlock->lastKey, keyInStt);




944,706✔





2075



  } else {










2076



    key = ascScan ? pBlock->firstKey : pBlock->lastKey;




532,553,833✔





2077



  }










2078












2079



  return key;




533,661,016✔





2080



}










2081












2082



static bool bufferDataInFileBlockGap(TSDBKEY keyInBuf, SFileDataBlockInfo* pBlock, STableBlockScanInfo* pScanInfo,




533,501,006✔





2083



                                     int32_t order) {










2084



  bool    ascScan = ASCENDING_TRAVERSE(order);




533,501,006✔





2085



  int64_t key = getBoarderKeyInFiles(pBlock, pScanInfo, order);




533,501,006✔





2086












2087



  return (ascScan && (keyInBuf.ts != TSKEY_INITIAL_VAL && keyInBuf.ts < key)) ||




1,067,251,726✔





2088



         (!ascScan && (keyInBuf.ts != TSKEY_INITIAL_VAL && keyInBuf.ts > key));




533,625,863✔





2089



}










2090












2091



static bool keyOverlapFileBlock(TSDBKEY key, SFileDataBlockInfo* pBlock, SVersionRange* pVerRange) {




566,758,671✔





2092



  return (key.ts >= pBlock->firstKey && key.ts <= pBlock->lastKey) && (pBlock->maxVer >= pVerRange->minVer) &&




592,316,475✔





2093



         (pBlock->minVer <= pVerRange->maxVer);




25,653,112✔





2094



}










2095












2096



static int32_t getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pBlockInfo,




566,801,061✔





2097



                                  STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, STsdbReader* pReader) {










2098



  int32_t             code = TSDB_CODE_SUCCESS;




566,801,061✔





2099



  int32_t             lino = 0;




566,801,061✔





2100



  SBrinRecord         rec = {0};




566,801,061✔





2101



  int32_t             neighborIndex = 0;




566,848,487✔





2102



  int32_t             order = 0;




566,849,935✔





2103



  SBlockLoadSuppInfo* pSupInfo = NULL;




566,849,935✔





2104



  SBrinRecord         pRecord;




566,849,935✔





2105



  bool                hasNeighbor = false;




566,872,729✔





2106












2107



  TSDB_CHECK_NULL(pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




566,860,807✔





2108



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




566,860,807✔





2109












2110



  order = pReader->info.order;




566,860,807✔





2111



  pSupInfo = &pReader->suppInfo;




566,862,290✔





2112












2113



  code = getNeighborBlockOfTable(&pReader->status.blockIter, pBlockInfo, pScanInfo, &neighborIndex, order, &rec,




566,914,291✔





2114



                                 pSupInfo, &hasNeighbor);










2115



  TSDB_CHECK_CODE(code, lino, _end);




566,872,462✔





2116












2117



  // overlap with neighbor










2118



  if (hasNeighbor) {




566,872,462✔





2119



    pInfo->overlapWithNeighborBlock =




221,359,840✔





2120



        overlapWithNeighborBlock2(pBlockInfo, &rec, order, pSupInfo->pk.type, pSupInfo->numOfPks);




221,374,606✔





2121



  }










2122












2123



  blockInfoToRecord(&pRecord, pBlockInfo, pSupInfo);




566,859,612✔





2124












2125



  // has duplicated ts of different version in this block










2126



  pInfo->hasDupTs = (pBlockInfo->numRow > pBlockInfo->count) || (pBlockInfo->count <= 0);




566,886,527✔





2127



  pInfo->overlapWithDelInfo = overlapWithDelSkyline(pScanInfo, &pRecord, order);




566,915,287✔





2128












2129



  // todo handle the primary key overlap case










2130



  if (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA) {




566,868,752✔





2131



    int64_t nextProcKeyInStt = pScanInfo->sttKeyInfo.nextProcKey.ts;




6,208,314✔





2132



    pInfo->overlapWithSttBlock = !(pBlockInfo->lastKey < nextProcKeyInStt || pBlockInfo->firstKey > nextProcKeyInStt);




6,208,314✔





2133



  }










2134












2135



  pInfo->moreThanCapcity = pBlockInfo->numRow > pReader->resBlockInfo.capacity;




566,875,190✔





2136



  pInfo->partiallyRequired = dataBlockPartiallyRequired(&pReader->info.window, &pReader->info.verRange, pBlockInfo);




566,887,070✔





2137



  pInfo->overlapWithKeyInBuf = keyOverlapFileBlock(keyInBuf, pBlockInfo, &pReader->info.verRange);




566,904,015✔





2138












2139



_end:




566,918,808✔





2140



  if (code != TSDB_CODE_SUCCESS) {




566,928,041✔





2141



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







2142



  }










2143



  return code;




566,928,041✔





2144



}










2145












2146



// 1. the version of all rows should be less than the endVersion










2147



// 2. current block should not overlap with next neighbor block










2148



// 3. current timestamp should not be overlap with each other










2149



// 4. output buffer should be large enough to hold all rows in current block










2150



// 5. delete info should not overlap with current block data










2151



// 6. current block should not contain the duplicated ts










2152



static int32_t fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pScanInfo,




550,136,518✔





2153



                                   TSDBKEY keyInBuf, bool* load) {










2154



  int32_t              code = TSDB_CODE_SUCCESS;




550,136,518✔





2155



  int32_t              lino = 0;




550,136,518✔





2156



  SDataBlockToLoadInfo info = {0};




550,136,518✔





2157












2158



  TSDB_CHECK_NULL(load, code, lino, _end, TSDB_CODE_INVALID_PARA);




550,245,923✔





2159












2160



  *load = false;




550,245,923✔





2161



  code = getBlockToLoadInfo(&info, pBlockInfo, pScanInfo, keyInBuf, pReader);




550,288,488✔





2162



  TSDB_CHECK_CODE(code, lino, _end);




550,293,114✔





2163












2164



  *load = (info.overlapWithNeighborBlock || info.hasDupTs || info.partiallyRequired || info.overlapWithKeyInBuf ||




1,100,562,712✔





2165



           info.moreThanCapcity || info.overlapWithDelInfo || info.overlapWithSttBlock);




1,100,562,712✔





2166












2167



  // log the reason why load the datablock for profile










2168



  if (*load) {




550,293,659✔





2169



    tsdbDebug("%p uid:%" PRIu64




16,625,800✔





2170



              " need to load the datablock, overlapneighbor:%d, hasDup:%d, partiallyRequired:%d, "










2171



              "overlapWithKey:%d, greaterThanBuf:%d, overlapWithDel:%d, overlapWithSttBlock:%d, %s",










2172



              pReader, pBlockInfo->uid, info.overlapWithNeighborBlock, info.hasDupTs, info.partiallyRequired,










2173



              info.overlapWithKeyInBuf, info.moreThanCapcity, info.overlapWithDelInfo, info.overlapWithSttBlock,










2174



              pReader->idStr);










2175



  }










2176












2177



_end:




550,235,011✔





2178



  if (code != TSDB_CODE_SUCCESS) {




550,139,467✔





2179



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







2180



  }










2181



  return code;




550,139,467✔





2182



}










2183












2184



static int32_t isCleanFileDataBlock(STsdbReader* pReader, SFileDataBlockInfo* pBlockInfo,




16,632,283✔





2185



                                    STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, bool* res) {










2186



  int32_t              code = TSDB_CODE_SUCCESS;




16,632,283✔





2187



  int32_t              lino = 0;




16,632,283✔





2188



  SDataBlockToLoadInfo info = {0};




16,632,283✔





2189












2190



  TSDB_CHECK_NULL(res, code, lino, _end, TSDB_CODE_INVALID_PARA);




16,632,283✔





2191












2192



  *res = false;




16,632,283✔





2193



  code = getBlockToLoadInfo(&info, pBlockInfo, pScanInfo, keyInBuf, pReader);




16,631,430✔





2194



  TSDB_CHECK_CODE(code, lino, _end);




16,632,283✔





2195












2196



  *res = !(info.overlapWithNeighborBlock || info.hasDupTs || info.overlapWithKeyInBuf || info.overlapWithDelInfo ||




19,720,462✔





2197



           info.overlapWithSttBlock);




3,088,179✔





2198












2199



_end:




16,632,283✔





2200



  if (code != TSDB_CODE_SUCCESS) {




16,632,283✔





2201



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







2202



  }










2203



  return code;




16,632,283✔





2204



}










2205












2206



static int32_t initRowMergeIfNeeded(STsdbReader* pReader, int64_t uid) {




2,147,483,647✔





2207



  int32_t     code = TSDB_CODE_SUCCESS;




2,147,483,647✔





2208



  int32_t     lino = 0;




2,147,483,647✔





2209



  SRowMerger* pMerger = NULL;




2,147,483,647✔





2210



  STSchema*   ps = NULL;




2,147,483,647✔





2211












2212



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





2213












2214



  pMerger = &pReader->status.merger;




2,147,483,647✔





2215












2216



  if (pMerger->pArray == NULL) {




2,147,483,647✔





2217



    ps = getTableSchemaImpl(pReader, uid);




326,420✔





2218



    TSDB_CHECK_NULL(ps, code, lino, _end, terrno);




326,420✔





2219



  }










2220












2221



_end:




2,147,483,647✔





2222



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





2223



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







2224



  }










2225



  return code;




2,147,483,647✔





2226



}










2227












2228



static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, int64_t endKey) {




2,147,483,647✔





2229



  int32_t      code = TSDB_CODE_SUCCESS;




2,147,483,647✔





2230



  int32_t      lino = 0;




2,147,483,647✔





2231



  int64_t      st = 0;




2,147,483,647✔





2232



  SSDataBlock* pBlock = NULL;




2,147,483,647✔





2233












2234



  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





2235












2236



  if (!(pBlockScanInfo->iiter.hasVal || pBlockScanInfo->iter.hasVal)) {




2,147,483,647✔





2237



    goto _end;




2,147,483,647✔





2238



  }










2239












2240



  code = initRowMergeIfNeeded(pReader, pBlockScanInfo->uid);




2,147,483,647✔





2241



  TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





2242












2243



  st = taosGetTimestampUs();




2,147,483,647✔





2244



  pBlock = pReader->resBlockInfo.pResBlock;




2,147,483,647✔





2245



  code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->resBlockInfo.capacity, pReader);




2,147,483,647✔





2246



  TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





2247












2248



  double el = (taosGetTimestampUs() - st) / 1000.0;




2,147,483,647✔





2249



  updateComposedBlockInfo(pReader, el, pBlockScanInfo);




2,147,483,647✔





2250












2251



  tsdbDebug("%p build data block from cache completed, elapsed time:%.2f ms, numOfRows:%" PRId64 ", brange:%" PRId64




2,147,483,647✔





2252



            " - %" PRId64 ", uid:%" PRIu64 ",  %s",










2253



            pReader, el, pBlock->info.rows, pBlock->info.window.skey, pBlock->info.window.ekey, pBlockScanInfo->uid,










2254



            pReader->idStr);










2255












2256



  pReader->cost.buildmemBlock += el;




2,147,483,647✔





2257












2258



_end:




2,147,483,647✔





2259



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





2260



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







2261



  }










2262



  return code;




2,147,483,647✔





2263



}










2264












2265



static int32_t tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pBlockData, SRowKey* pKey,




1,593,313,624✔





2266



                                               SFileBlockDumpInfo* pDumpInfo, bool* copied) {










2267



  // opt version










2268



  // 1. it is not a border point










2269



  // 2. the direct next point is not an duplicated timestamp










2270



  int32_t code = TSDB_CODE_SUCCESS;




1,593,313,624✔





2271



  int32_t lino = 0;




1,593,313,624✔





2272



  bool    asc = false;




1,593,313,624✔





2273












2274



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,593,313,624✔





2275



  TSDB_CHECK_NULL(pDumpInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,593,313,624✔





2276



  TSDB_CHECK_NULL(copied, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,593,313,624✔





2277












2278



  *copied = false;




1,593,313,624✔





2279



  asc = (pReader->info.order == TSDB_ORDER_ASC);




1,593,313,624✔





2280



  if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && asc) || (pDumpInfo->rowIndex > 0 && (!asc))) {




1,593,313,624✔





2281



    int32_t step = ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1;




1,592,276,473✔





2282












2283



    SRowKey nextRowKey;




1,592,276,473✔





2284



    TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,592,276,473✔





2285



    tColRowGetKey(pBlockData, pDumpInfo->rowIndex + step, &nextRowKey);




1,592,276,473✔





2286












2287



    if (pkCompEx(pKey, &nextRowKey) != 0) {  // merge is not needed




1,592,276,473✔





2288



      code = doAppendRowFromFileBlock(pReader->resBlockInfo.pResBlock, pReader, pBlockData, pDumpInfo->rowIndex);




1,592,276,473✔





2289



      TSDB_CHECK_CODE(code, lino, _end);




1,592,276,473✔





2290



      pDumpInfo->rowIndex += step;




1,592,276,473✔





2291



      *copied = true;




1,592,276,473✔





2292



    }










2293



  }










2294












2295



_end:




1,037,151✔





2296



  if (code != TSDB_CODE_SUCCESS) {




1,593,313,624✔





2297



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







2298



  }










2299



  return code;




1,593,313,624✔





2300



}










2301












2302



static int32_t nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, int32_t pkSrcSlot,




2,147,483,647✔





2303



                                    SVersionRange* pVerRange) {










2304



  int32_t  code = TSDB_CODE_SUCCESS;




2,147,483,647✔





2305



  int32_t  lino = 0;




2,147,483,647✔





2306



  int32_t  order = 0;




2,147,483,647✔





2307



  int32_t  step = 0;




2,147,483,647✔





2308



  SRowKey* pNextProc = NULL;




2,147,483,647✔





2309












2310



  TSDB_CHECK_NULL(pSttBlockReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





2311



  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





2312












2313



  order = pSttBlockReader->order;




2,147,483,647✔





2314



  step = ASCENDING_TRAVERSE(order) ? 1 : -1;




2,147,483,647✔





2315



  pNextProc = &pScanInfo->sttKeyInfo.nextProcKey;




2,147,483,647✔





2316












2317



  while (1) {




857,744,462✔





2318



    bool hasVal = false;




2,147,483,647✔





2319



    code = tMergeTreeNext(&pSttBlockReader->mergeTree, &hasVal);




2,147,483,647✔





2320



    if (code) {




2,147,483,647✔





2321



      tsdbError("failed to iter the next row in stt-file merge tree, code:%s, %s", tstrerror(code),




×







2322



                pSttBlockReader->mergeTree.idStr);










2323



      TSDB_CHECK_CODE(code, lino, _end);




95,852✔





2324



    }










2325












2326



    if (!hasVal) {  // the next value will be the accessed key in stt




2,147,483,647✔





2327



      pScanInfo->sttKeyInfo.status = STT_FILE_NO_DATA;




2,147,483,647✔





2328












2329



      // next file, the timestamps in the next file must be greater than those in current










2330



      pNextProc->ts += step;




2,147,483,647✔





2331



      if (pSttBlockReader->numOfPks > 0) {




2,147,483,647✔





2332



        if (IS_NUMERIC_TYPE(pNextProc->pks[0].type)) {




1,216,138,801✔





2333



          VALUE_SET_TRIVIAL_DATUM(pNextProc->pks, INT64_MAX);




805,120,881✔





2334



        } else {










2335



          memset(pNextProc->pks[0].pData, 0, pNextProc->pks[0].nData);




410,954,167✔





2336



        }










2337



      }










2338



      goto _end;




2,147,483,647✔





2339



    }










2340












2341



    TSDBROW* pRow = tMergeTreeGetRow(&pSttBlockReader->mergeTree);




2,147,483,647✔





2342



    int64_t  key = pRow->pBlockData->aTSKEY[pRow->iRow];




2,147,483,647✔





2343



    int64_t  ver = pRow->pBlockData->aVersion[pRow->iRow];




2,147,483,647✔





2344












2345



    if (pSttBlockReader->numOfPks == 0) {




2,147,483,647✔





2346



      pSttBlockReader->currentKey.ts = key;




2,147,483,647✔





2347



    } else {










2348



      tColRowGetKeyDeepCopy(pRow->pBlockData, pRow->iRow, pkSrcSlot, &pSttBlockReader->currentKey);




2,147,483,647✔





2349



    }










2350












2351



    tColRowGetKeyDeepCopy(pRow->pBlockData, pRow->iRow, pkSrcSlot, pNextProc);




2,147,483,647✔





2352












2353



    if (pScanInfo->delSkyline != NULL && TARRAY_SIZE(pScanInfo->delSkyline) > 0) {




2,147,483,647✔





2354



      bool dropped = false;




2,147,483,647✔





2355



      code = hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->sttBlockDelIndex, key, ver, order, pVerRange,




2,147,483,647✔





2356



                            pSttBlockReader->numOfPks > 0, &dropped);




2,147,483,647✔





2357



      TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





2358



      if (!dropped) {




2,147,483,647✔





2359



        pScanInfo->sttKeyInfo.status = STT_FILE_HAS_DATA;




1,396,760,976✔





2360



        goto _end;




1,398,278,337✔





2361



      }










2362



    } else {










2363



      pScanInfo->sttKeyInfo.status = STT_FILE_HAS_DATA;




2,147,483,647✔





2364



      goto _end;




2,147,483,647✔





2365



    }










2366



  }










2367












2368



_end:




2,147,483,647✔





2369



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





2370



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







2371



  }










2372



  return code;




2,147,483,647✔





2373



}










2374












2375



static void doPinSttBlock(SSttBlockReader* pSttBlockReader) { tMergeTreePinSttBlock(&pSttBlockReader->mergeTree); }




2,147,483,647✔





2376












2377



static void doUnpinSttBlock(SSttBlockReader* pSttBlockReader) { tMergeTreeUnpinSttBlock(&pSttBlockReader->mergeTree); }




2,147,483,647✔





2378












2379



static int32_t tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SSttBlockReader* pSttBlockReader,




2,147,483,647✔





2380



                                              STableBlockScanInfo* pScanInfo, SRowKey* pSttKey, STsdbReader* pReader,










2381



                                              bool* copied) {










2382



  int32_t  code = TSDB_CODE_SUCCESS;




2,147,483,647✔





2383



  int32_t  lino = 0;




2,147,483,647✔





2384



  SRowKey* pNext = NULL;




2,147,483,647✔





2385












2386



  TSDB_CHECK_NULL(pSttBlockReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





2387



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





2388



  TSDB_CHECK_NULL(copied, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





2389












2390



  *copied = false;




2,147,483,647✔





2391












2392



  // avoid the fetch next row replace the referenced stt block in buffer










2393



  doPinSttBlock(pSttBlockReader);




2,147,483,647✔





2394



  code = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pReader->suppInfo.pkSrcSlot, &pReader->info.verRange);




2,147,483,647✔





2395



  doUnpinSttBlock(pSttBlockReader);




2,147,483,647✔





2396



  TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





2397












2398



  if (hasDataInSttBlock(pScanInfo)) {




2,147,483,647✔





2399



    pNext = getCurrentKeyInSttBlock(pSttBlockReader);




2,147,483,647✔





2400



    if (pkCompEx(pSttKey, pNext) != 0) {




2,147,483,647✔





2401



      code = doAppendRowFromFileBlock(pReader->resBlockInfo.pResBlock, pReader, fRow->pBlockData, fRow->iRow);




2,147,483,647✔





2402



      *copied = (code == TSDB_CODE_SUCCESS);




2,147,483,647✔





2403



      TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





2404



    }










2405



  } else {










2406



    code = doAppendRowFromFileBlock(pReader->resBlockInfo.pResBlock, pReader, fRow->pBlockData, fRow->iRow);




1,834,158,844✔





2407



    *copied = (code == TSDB_CODE_SUCCESS);




1,873,320,402✔





2408



    TSDB_CHECK_CODE(code, lino, _end);




1,873,122,386✔





2409



  }










2410












2411



_end:




1,873,122,386✔





2412



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





2413



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







2414



  }










2415



  return code;




2,147,483,647✔





2416



}










2417












2418



static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* pReader, uint64_t uid) {










2419



  int32_t   code = TSDB_CODE_SUCCESS;




2,147,483,647✔





2420



  int32_t   lino = 0;




2,147,483,647✔





2421



  STSchema* ps = NULL;




2,147,483,647✔





2422



  void**    p = NULL;




2,147,483,647✔





2423












2424



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





2425












2426



  // always set the newest schema version in pReader->info.pSchema










2427



  if (pReader->info.pSchema == NULL) {




2,147,483,647✔





2428



    ps = getTableSchemaImpl(pReader, uid);




×







2429



    TSDB_CHECK_NULL(ps, code, lino, _end, terrno);




×







2430



  }










2431












2432



  if (pReader->info.pSchema && sversion == pReader->info.pSchema->version) {




2,147,483,647✔





2433



    ps = pReader->info.pSchema;




2,147,483,647✔





2434



    goto _end;




2,147,483,647✔





2435



  }










2436












2437



  p = tSimpleHashGet(pReader->pSchemaMap, &sversion, sizeof(sversion));




2,147,483,647✔





2438



  if (p != NULL) {




2,147,483,647✔





2439



    ps = *(STSchema**)p;




2,147,483,647✔





2440



    goto _end;




2,147,483,647✔





2441



  }










2442












2443



  code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, uid, sversion, &ps);




85,621,811✔





2444



  TSDB_CHECK_CODE(code, lino, _end);




85,657,499✔





2445












2446



  code = tSimpleHashPut(pReader->pSchemaMap, &sversion, sizeof(sversion), &ps, POINTER_BYTES);




85,657,499✔





2447



  TSDB_CHECK_CODE(code, lino, _end);




85,653,969✔





2448












2449



_end:




2,147,483,647✔





2450



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





2451



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







2452



    terrno = code;




×







2453



    return NULL;




×







2454



  }










2455



  return ps;




2,147,483,647✔





2456



}










2457












2458



// deep copy the primary key, to avoid ref invalid buffer if the primary key is varchar or something.










2459



static int32_t primaryKeyDup(SRowKey* pKey) {




2,147,483,647✔





2460



  if (pKey->numOfPKs > 0) {




2,147,483,647✔





2461



    if (!IS_NUMERIC_TYPE(pKey->pks[0].type)) {




34,897,920✔





2462



      void* p = taosMemoryMalloc(pKey->pks[0].nData);




34,897,920✔





2463



      if (p == NULL)  {




34,897,920✔





2464



        tsdbError("%s failed at %s:%d since %s", __func__, __FILE__, __LINE__, tstrerror(terrno));




×







2465



        // memory allocation failed, set pData to NULL to avoid freeing original buffer










2466



        pKey->pks[0].pData = NULL;




×







2467



        return terrno;




×







2468



      }










2469












2470



      TAOS_MEMCPY(p, pKey->pks[0].pData, pKey->pks[0].nData);




34,897,920✔





2471



      pKey->pks[0].pData = p;




34,897,920✔





2472



    }










2473



  }










2474












2475



  return 0;




2,147,483,647✔





2476



}










2477












2478



static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow,




2,147,483,647✔





2479



                                     SIterInfo* pIter, SSttBlockReader* pSttBlockReader) {










2480



  int32_t             code = TSDB_CODE_SUCCESS;




2,147,483,647✔





2481



  int32_t             lino = 0;




2,147,483,647✔





2482



  SRowMerger*         pMerger = NULL;




2,147,483,647✔





2483



  SRow*               pTSRow = NULL;




2,147,483,647✔





2484



  SBlockData*         pBlockData = NULL;




2,147,483,647✔





2485



  SFileBlockDumpInfo* pDumpInfo = NULL;




2,147,483,647✔





2486



  SRowKey*            pSttKey = NULL;




2,147,483,647✔





2487



  int32_t             pkSrcSlot = 0;




2,147,483,647✔





2488



  SRowKey             k = {0};




2,147,483,647✔





2489



  STSchema*           pSchema = NULL;




2,147,483,647✔





2490












2491



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





2492



  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





2493



  TSDB_CHECK_NULL(pRow, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





2494












2495



  pMerger = &pReader->status.merger;




2,147,483,647✔





2496



  pMerger->arg = pReader->pTsdb->pVnode->pBse;




2,147,483,647✔





2497












2498



  pBlockData = &pReader->status.fileBlockData;




2,147,483,647✔





2499



  pDumpInfo = &pReader->status.fBlockDumpInfo;




2,147,483,647✔





2500



  pkSrcSlot = pReader->suppInfo.pkSrcSlot;




2,147,483,647✔





2501












2502



  if (hasDataInSttBlock(pBlockScanInfo) && (!pBlockScanInfo->cleanSttBlocks)) {




2,147,483,647✔





2503



    pSttKey = getCurrentKeyInSttBlock(pSttBlockReader);




2,147,483,647✔





2504



  }










2505












2506



  tRowGetKeyEx(pRow, &k);




2,147,483,647✔





2507












2508



  if (pRow->type == TSDBROW_ROW_FMT) {




2,147,483,647✔





2509



    pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid);




2,147,483,647✔





2510



    TSDB_CHECK_NULL(pSchema, code, lino, _end, terrno);




2,147,483,647✔





2511



  }










2512












2513



  SRowKey* pfKey = &(SRowKey){0};




2,147,483,647✔





2514



  if (hasDataInFileBlock(pBlockData, pDumpInfo)) {




2,147,483,647✔





2515



    tColRowGetKey(pBlockData, pDumpInfo->rowIndex, pfKey);




2,147,483,647✔





2516



    code = primaryKeyDup(pfKey);




2,147,483,647✔





2517



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





2518



  } else {










2519



    pfKey = NULL;




2,147,483,647✔





2520



  }










2521












2522



  TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);




2,147,483,647✔





2523












2524



  // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized










2525



  code = initRowMergeIfNeeded(pReader, pBlockScanInfo->uid);




2,147,483,647✔





2526



  TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





2527












2528



  SRowKey minKey = k;




2,147,483,647✔





2529



  if (pReader->info.order == TSDB_ORDER_ASC) {




2,147,483,647✔





2530



    if (pfKey != NULL && pkCompEx(pfKey, &minKey) < 0) {




2,147,483,647✔





2531



      minKey = *pfKey;




2,147,483,647✔





2532



    }










2533












2534



    if (pSttKey != NULL && pkCompEx(pSttKey, &minKey) < 0) {




2,147,483,647✔





2535



      minKey = *pSttKey;




2,147,483,647✔





2536



    }










2537



  } else {










2538



    if (pfKey != NULL && pkCompEx(pfKey, &minKey) > 0) {




1,642,475,609✔





2539



      minKey = *pfKey;




247,069,598✔





2540



    }










2541












2542



    if (pSttKey != NULL && pkCompEx(pSttKey, &minKey) > 0) {




1,801,995,840✔





2543



      minKey = *pSttKey;




56,777,445✔





2544



    }










2545



  }










2546












2547



  // copy the last key before the time of stt reader loading the next stt block, in which the underlying data block may










2548



  // be changed, resulting in the corresponding changing of the value of sttRowKey










2549



  tRowKeyAssign(&pBlockScanInfo->lastProcKey, &minKey);




2,147,483,647✔





2550












2551



  // file block ---> stt block -----> mem










2552



  if (pkCompEx(&pBlockScanInfo->lastProcKey, pfKey) == 0) {




2,147,483,647✔





2553



    code = tsdbRowMergerAdd(pMerger, &fRow, NULL);




2,147,483,647✔





2554



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





2555












2556



    code = doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pfKey, pReader);




2,147,483,647✔





2557



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





2558



  }










2559












2560



  if (pkCompEx(&pBlockScanInfo->lastProcKey, pSttKey) == 0) {




2,147,483,647✔





2561



    TSDBROW* fRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree);




2,147,483,647✔





2562



    code = tsdbRowMergerAdd(pMerger, fRow1, NULL);




2,147,483,647✔





2563



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





2564



    code = doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange,




2,147,483,647✔





2565



                                 pReader->idStr);




2,147,483,647✔





2566



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





2567



  }










2568












2569



  if (pkCompEx(&pBlockScanInfo->lastProcKey, &k) == 0) {




2,147,483,647✔





2570



    code = tsdbRowMergerAdd(pMerger, pRow, pSchema);




2,147,483,647✔





2571



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





2572












2573



    code = doMergeRowsInBuf(pIter, pBlockScanInfo->uid, &k, pBlockScanInfo->delSkyline, pReader);




2,147,483,647✔





2574



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





2575



  }










2576












2577



  code = tsdbRowMergerGetRow(pMerger, &pTSRow);




2,147,483,647✔





2578



  TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





2579












2580



  code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo);




2,147,483,647✔





2581












2582



  taosMemoryFreeClear(pTSRow);




2,147,483,647✔





2583



  tsdbRowMergerClear(pMerger);




2,147,483,647✔





2584



  TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





2585












2586



_end:




2,147,483,647✔





2587



  clearRowKey(pfKey);




2,147,483,647✔





2588



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





2589



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







2590



  }










2591



  return code;




2,147,483,647✔





2592



}










2593












2594



static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* pSttBlockReader, SRowKey* pKey,




2,147,483,647✔





2595



                                         STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) {










2596



  int32_t             code = TSDB_CODE_SUCCESS;




2,147,483,647✔





2597



  int32_t             lino = 0;




2,147,483,647✔





2598



  SFileBlockDumpInfo* pDumpInfo = NULL;




2,147,483,647✔





2599



  SRowMerger*         pMerger = NULL;




2,147,483,647✔





2600



  SRow*               pTSRow = NULL;




2,147,483,647✔





2601



  int32_t             pkSrcSlot = 0;




2,147,483,647✔





2602












2603



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





2604



  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





2605



  TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





2606












2607



  pDumpInfo = &pReader->status.fBlockDumpInfo;




2,147,483,647✔





2608



  pMerger = &pReader->status.merger;




2,147,483,647✔





2609



  pMerger->arg = pReader->pTsdb->pVnode->pBse;




2,147,483,647✔





2610












2611



  pkSrcSlot = pReader->suppInfo.pkSrcSlot;




2,147,483,647✔





2612












2613



  // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized










2614



  code = initRowMergeIfNeeded(pReader, pBlockScanInfo->uid);




2,147,483,647✔





2615



  TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





2616












2617



  bool dataInDataFile = hasDataInFileBlock(pBlockData, pDumpInfo);




2,147,483,647✔





2618



  bool dataInSttFile = hasDataInSttBlock(pBlockScanInfo);




2,147,483,647✔





2619












2620



  if (dataInDataFile && (!dataInSttFile)) {




2,147,483,647✔





2621



    // no stt file block available, only data block exists










2622



    code = mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader);




1,592,147,162✔





2623



    TSDB_CHECK_CODE(code, lino, _end);




1,592,147,162✔





2624



  } else if ((!dataInDataFile) && dataInSttFile) {




2,147,483,647✔





2625



    // no data in data file exists










2626



    code = mergeRowsInSttBlocks(pSttBlockReader, pBlockScanInfo, pReader);




2,147,483,647✔





2627



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





2628



  } else if (pBlockScanInfo->cleanSttBlocks && pReader->info.execMode == READER_EXEC_ROWS) {




110,946✔





2629



    // opt model for count data in stt file, which is not overlap with data blocks in files.










2630



    code = mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader);




×







2631



    TSDB_CHECK_CODE(code, lino, _end);




×







2632



  } else {










2633



    TSDB_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,089,214✔





2634



    // row in both stt file blocks and data file blocks










2635



    TSDBROW  fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);




2,089,214✔





2636



    SRowKey* pSttKey = getCurrentKeyInSttBlock(pSttBlockReader);




2,089,214✔





2637












2638



    int32_t ret = pkCompEx(pKey, pSttKey);




2,089,214✔





2639












2640



    if (ASCENDING_TRAVERSE(pReader->info.order)) {




2,089,214✔





2641



      if (ret < 0) {  // asc




2,089,214✔





2642



        code = mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader);




1,166,462✔





2643



        TSDB_CHECK_CODE(code, lino, _end);




1,166,462✔





2644



      } else if (ret > 0) {




922,752✔





2645



        code = mergeRowsInSttBlocks(pSttBlockReader, pBlockScanInfo, pReader);




352,368✔





2646



        TSDB_CHECK_CODE(code, lino, _end);




352,368✔





2647



      }










2648



    } else {  // desc










2649



      if (ret > 0) {




×







2650



        code = mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader);




×







2651



        TSDB_CHECK_CODE(code, lino, _end);




×







2652



      } else if (ret < 0) {




×







2653



        code = mergeRowsInSttBlocks(pSttBlockReader, pBlockScanInfo, pReader);




×







2654



        TSDB_CHECK_CODE(code, lino, _end);




×







2655



      }










2656



    }










2657



    if (ret != 0) {




2,089,214✔





2658



      goto _end;




1,518,830✔





2659



    }










2660












2661



    // pKey == pSttKey










2662



    tRowKeyAssign(&pBlockScanInfo->lastProcKey, pKey);




570,384✔





2663












2664



    // the following for key == sttKey->key.ts










2665



    // file block ------> stt  block










2666












2667



    code = tsdbRowMergerAdd(pMerger, &fRow, NULL);




570,384✔





2668



    TSDB_CHECK_CODE(code, lino, _end);




570,384✔





2669












2670



    code = doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader);




570,384✔





2671



    TSDB_CHECK_CODE(code, lino, _end);




570,384✔





2672












2673



    TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree);




570,384✔





2674



    code = tsdbRowMergerAdd(pMerger, pRow1, NULL);




570,384✔





2675



    TSDB_CHECK_CODE(code, lino, _end);




570,384✔





2676












2677



    // pSttKey will be changed when sttBlockReader iterates to the next row, so use pKey instead.










2678



    code = doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange,




570,384✔





2679



                                 pReader->idStr);




570,384✔





2680



    TSDB_CHECK_CODE(code, lino, _end);




570,384✔





2681












2682



    code = tsdbRowMergerGetRow(pMerger, &pTSRow);




570,384✔





2683



    TSDB_CHECK_CODE(code, lino, _end);




570,384✔





2684












2685



    code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo);




570,384✔





2686












2687



    taosMemoryFreeClear(pTSRow);




570,384✔





2688



    tsdbRowMergerClear(pMerger);




570,384✔





2689



    TSDB_CHECK_CODE(code, lino, _end);




570,384✔





2690



  }










2691












2692



_end:




2,147,483,647✔





2693



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





2694



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







2695



  }










2696



  return code;




2,147,483,647✔





2697



}










2698












2699



static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData,




295,687✔





2700



                                     SSttBlockReader* pSttBlockReader) {










2701



  int32_t             code = TSDB_CODE_SUCCESS;




295,687✔





2702



  int32_t             lino = 0;




295,687✔





2703



  SRowMerger*         pMerger = NULL;




295,687✔





2704



  SRow*               pTSRow = NULL;




295,687✔





2705



  SFileBlockDumpInfo* pDumpInfo = NULL;




295,687✔





2706



  SArray*             pDelList = NULL;




295,687✔





2707



  int32_t             pkSrcSlot = 0;




295,687✔





2708



  TSDBROW*            pRow = NULL;




295,687✔





2709



  TSDBROW*            piRow = NULL;




295,687✔





2710



  SRowKey*            pSttKey = NULL;




295,687✔





2711












2712



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




295,687✔





2713



  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




295,687✔





2714



  TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA);




295,687✔





2715












2716



  pMerger = &pReader->status.merger;




295,687✔





2717



  pDumpInfo = &pReader->status.fBlockDumpInfo;




295,687✔





2718



  pDelList = pBlockScanInfo->delSkyline;




295,687✔





2719



  pkSrcSlot = pReader->suppInfo.pkSrcSlot;




295,687✔





2720












2721



  code = getValidMemRow(&pBlockScanInfo->iter, pDelList, pReader, &pRow);




295,687✔





2722



  TSDB_CHECK_CODE(code, lino, _end);




295,687✔





2723



  code = getValidMemRow(&pBlockScanInfo->iiter, pDelList, pReader, &piRow);




295,687✔





2724



  TSDB_CHECK_CODE(code, lino, _end);




295,687✔





2725












2726



  if (hasDataInSttBlock(pBlockScanInfo) && (!pBlockScanInfo->cleanSttBlocks)) {




295,687✔





2727



    pSttKey = getCurrentKeyInSttBlock(pSttBlockReader);




295,687✔





2728



  }










2729












2730



  SRowKey* pfKey = &(SRowKey){0};




295,687✔





2731



  if (hasDataInFileBlock(pBlockData, pDumpInfo)) {




295,687✔





2732



    tColRowGetKey(pBlockData, pDumpInfo->rowIndex, pfKey);




×







2733



  } else {










2734



    pfKey = NULL;




295,687✔





2735



  }










2736












2737



  SRowKey k = {0}, ik = {0};




295,687✔





2738



  tRowGetKeyEx(pRow, &k);




295,687✔





2739



  tRowGetKeyEx(piRow, &ik);




295,687✔





2740












2741



  STSchema* pSchema = NULL;




295,687✔





2742



  if (pRow->type == TSDBROW_ROW_FMT) {




295,687✔





2743



    pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid);




295,687✔





2744



    TSDB_CHECK_NULL(pSchema, code, lino, _end, terrno);




295,687✔





2745



  }










2746












2747



  STSchema* piSchema = NULL;




295,687✔





2748



  if (piRow->type == TSDBROW_ROW_FMT) {




295,687✔





2749



    piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid);




295,687✔





2750



    TSDB_CHECK_NULL(piSchema, code, lino, _end, terrno);




295,687✔





2751



  }










2752












2753



  // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized










2754



  code = initRowMergeIfNeeded(pReader, pBlockScanInfo->uid);




295,687✔





2755



  TSDB_CHECK_CODE(code, lino, _end);




295,687✔





2756












2757



  SRowKey minKey = k;




295,687✔





2758



  if (ASCENDING_TRAVERSE(pReader->info.order)) {




295,687✔





2759



    if (pkCompEx(&ik, &minKey) < 0) {  // minKey > ik.key.ts) {




176,696✔





2760



      minKey = ik;




9,512✔





2761



    }










2762












2763



    if ((pfKey != NULL) && (pkCompEx(pfKey, &minKey) < 0)) {




176,696✔





2764



      minKey = *pfKey;




×







2765



    }










2766












2767



    if ((pSttKey != NULL) && (pkCompEx(pSttKey, &minKey) < 0)) {




353,392✔





2768



      minKey = *pSttKey;




7,134✔





2769



    }










2770



  } else {










2771



    if (pkCompEx(&ik, &minKey) > 0) {




118,991✔





2772



      minKey = ik;




×







2773



    }










2774












2775



    if ((pfKey != NULL) && (pkCompEx(pfKey, &minKey) > 0)) {




118,991✔





2776



      minKey = *pfKey;




×







2777



    }










2778












2779



    if ((pSttKey != NULL) && (pkCompEx(pSttKey, &minKey) > 0)) {




237,982✔





2780



      minKey = *pSttKey;




×







2781



    }










2782



  }










2783












2784



  tRowKeyAssign(&pBlockScanInfo->lastProcKey, &minKey);




295,687✔





2785












2786



  // file block -----> stt block -----> imem -----> mem










2787



  if (pkCompEx(&pBlockScanInfo->lastProcKey, pfKey) == 0) {




591,374✔





2788



    TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);




×







2789



    code = tsdbRowMergerAdd(pMerger, &fRow, NULL);




×







2790



    TSDB_CHECK_CODE(code, lino, _end);




×







2791












2792



    code = doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pfKey, pReader);




×







2793



    TSDB_CHECK_CODE(code, lino, _end);




×







2794



  }










2795












2796



  if (pkCompEx(&pBlockScanInfo->lastProcKey, pSttKey) == 0) {




591,374✔





2797



    TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree);




175,320✔





2798



    code = tsdbRowMergerAdd(pMerger, pRow1, NULL);




175,320✔





2799



    TSDB_CHECK_CODE(code, lino, _end);




175,320✔





2800












2801



    code = doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange,




175,320✔





2802



                                 pReader->idStr);




175,320✔





2803



    TSDB_CHECK_CODE(code, lino, _end);




175,320✔





2804



  }










2805












2806



  if (pkCompEx(&pBlockScanInfo->lastProcKey, &ik) == 0) {




591,374✔





2807



    code = tsdbRowMergerAdd(pMerger, piRow, piSchema);




222,527✔





2808



    TSDB_CHECK_CODE(code, lino, _end);




222,527✔





2809












2810



    code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, &ik, pBlockScanInfo->delSkyline, pReader);




222,527✔





2811



    TSDB_CHECK_CODE(code, lino, _end);




222,527✔





2812



  }










2813












2814



  if (pkCompEx(&pBlockScanInfo->lastProcKey, &k) == 0) {




591,374✔





2815



    code = tsdbRowMergerAdd(pMerger, pRow, pSchema);




286,175✔





2816



    TSDB_CHECK_CODE(code, lino, _end);




286,175✔





2817












2818



    code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, &k, pBlockScanInfo->delSkyline, pReader);




286,175✔





2819



    TSDB_CHECK_CODE(code, lino, _end);




286,175✔





2820



  }










2821












2822



  code = tsdbRowMergerGetRow(pMerger, &pTSRow);




295,687✔





2823



  TSDB_CHECK_CODE(code, lino, _end);




295,687✔





2824












2825



  code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo);




295,687✔





2826












2827



  taosMemoryFreeClear(pTSRow);




295,687✔





2828



  tsdbRowMergerClear(pMerger);




295,687✔





2829



  TSDB_CHECK_CODE(code, lino, _end);




295,687✔





2830












2831



_end:




295,687✔





2832



  if (code != TSDB_CODE_SUCCESS) {




295,687✔





2833



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







2834



  }










2835



  return code;




295,687✔





2836



}










2837












2838



int32_t doInitMemDataIter(STsdbReader* pReader, STbData** pData, STableBlockScanInfo* pBlockScanInfo, STsdbRowKey* pKey,




2,147,483,647✔





2839



                          SMemTable* pMem, SIterInfo* pIter, const char* type) {










2840



  int32_t code = TSDB_CODE_SUCCESS;




2,147,483,647✔





2841



  int32_t lino = 0;




2,147,483,647✔





2842



  int32_t backward = false;




2,147,483,647✔





2843












2844



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





2845



  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





2846



  TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





2847



  TSDB_CHECK_NULL(type, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





2848












2849



  backward = (!ASCENDING_TRAVERSE(pReader->info.order));




2,147,483,647✔





2850



  pIter->hasVal = false;




2,147,483,647✔





2851












2852



  if (pMem != NULL) {




2,147,483,647✔





2853



    TSDB_CHECK_NULL(pData, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





2854



    *pData = tsdbGetTbDataFromMemTable(pMem, pReader->info.suid, pBlockScanInfo->uid);




2,147,483,647✔





2855












2856



    if ((*pData) != NULL) {




2,147,483,647✔





2857



      code = tsdbTbDataIterCreate((*pData), pKey, backward, &pIter->iter);




2,147,483,647✔





2858



      if (code == TSDB_CODE_SUCCESS) {




2,147,483,647✔





2859



        pIter->hasVal = (tsdbTbDataIterGet(pIter->iter) != NULL);




2,147,483,647✔





2860












2861



        tsdbDebug("%p uid:%" PRIu64 ", check data in %s from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64




2,147,483,647✔





2862



                  "-%" PRId64 " %s",










2863



                  pReader, pBlockScanInfo->uid, type, pKey->key.ts, pReader->info.order, (*pData)->minKey,










2864



                  (*pData)->maxKey, pReader->idStr);










2865



      } else {










2866



        tsdbError("%p uid:%" PRIu64 ", failed to create iterator for %s, code:%s, %s", pReader, pBlockScanInfo->uid,




×







2867



                  type, tstrerror(code), pReader->idStr);










2868



        TSDB_CHECK_CODE(code, lino, _end);




×







2869



      }










2870



    }










2871



  } else {










2872



    tsdbDebug("%p uid:%" PRIu64 ", no data in %s, %s", pReader, pBlockScanInfo->uid, type, pReader->idStr);




2,147,483,647✔





2873



  }










2874












2875



_end:




2,147,483,647✔





2876



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





2877



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







2878



  }










2879



  return code;




2,147,483,647✔





2880



}










2881












2882



static int32_t doForwardDataIter(SRowKey* pKey, SIterInfo* pIter, STableBlockScanInfo* pBlockScanInfo,




917,617,674✔





2883



                                 STsdbReader* pReader) {










2884



  int32_t  code = TSDB_CODE_SUCCESS;




917,617,674✔





2885



  int32_t  lino = 0;




917,617,674✔





2886



  SRowKey  rowKey = {0};




917,617,674✔





2887



  TSDBROW* pRow = NULL;




917,731,604✔





2888












2889



  while (1) {




×







2890



    code = getValidMemRow(pIter, pBlockScanInfo->delSkyline, pReader, &pRow);




917,708,080✔





2891



    TSDB_CHECK_CODE(code, lino, _end);




917,653,503✔





2892



    if (!pIter->hasVal) {




917,653,503✔





2893



      break;




511,314,702✔





2894



    }










2895












2896



    tRowGetKeyEx(pRow, &rowKey);




406,402,446✔





2897



    int32_t ret = pkCompEx(pKey, &rowKey);




406,486,211✔





2898



    if (ret == 0) {




406,486,211✔





2899



      pIter->hasVal = tsdbTbDataIterNext(pIter->iter);




×







2900



    } else {










2901



      break;




406,486,211✔





2902



    }










2903



  }










2904












2905



_end:




917,800,913✔





2906



  if (code != TSDB_CODE_SUCCESS) {




917,706,389✔





2907



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







2908



  }










2909



  return code;




917,706,389✔





2910



}










2911












2912



// handle the open interval issue. Find the first row key that is greater than the given one.










2913



static int32_t forwardDataIter(SRowKey* pKey, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) {




458,810,059✔





2914



  int32_t code = TSDB_CODE_SUCCESS;




458,810,059✔





2915



  int32_t lino = 0;




458,810,059✔





2916












2917



  TSDB_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA);




458,810,059✔





2918



  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




458,810,059✔





2919












2920



  code = doForwardDataIter(pKey, &pBlockScanInfo->iter, pBlockScanInfo, pReader);




458,810,059✔





2921



  TSDB_CHECK_CODE(code, lino, _end);




458,806,117✔





2922



  code = doForwardDataIter(pKey, &pBlockScanInfo->iiter, pBlockScanInfo, pReader);




458,806,117✔





2923



  TSDB_CHECK_CODE(code, lino, _end);




458,993,566✔





2924












2925



_end:




458,993,566✔





2926



  if (code != TSDB_CODE_SUCCESS) {




458,993,566✔





2927



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







2928



  }










2929



  return code;




458,793,833✔





2930



}










2931












2932



static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) {




2,147,483,647✔





2933



  int32_t        code = TSDB_CODE_SUCCESS;




2,147,483,647✔





2934



  int32_t        lino = 0;




2,147,483,647✔





2935



  STbData*       d = NULL;




2,147,483,647✔





2936



  STbData*       di = NULL;




2,147,483,647✔





2937



  bool           asc = false;




2,147,483,647✔





2938



  bool           forward = true;




2,147,483,647✔





2939



  STsdbReadSnap* pSnap = NULL;




2,147,483,647✔





2940



  STimeWindow*   pWindow = NULL;




2,147,483,647✔





2941



  STsdbRowKey    startKey;




2,147,483,647✔





2942












2943



  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





2944



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





2945












2946



  asc = ASCENDING_TRAVERSE(pReader->info.order);




2,147,483,647✔





2947



  pSnap = pReader->pReadSnap;




2,147,483,647✔





2948



  pWindow = &pReader->info.window;




2,147,483,647✔





2949












2950



  if (pBlockScanInfo->iterInit) {




2,147,483,647✔





2951



    goto _end;




2,147,483,647✔





2952



  }










2953












2954



  startKey.key = pBlockScanInfo->lastProcKey;




2,147,483,647✔





2955



  startKey.version = asc ? pReader->info.verRange.minVer : pReader->info.verRange.maxVer;




2,147,483,647✔





2956



  if ((asc && (startKey.key.ts < pWindow->skey)) || ((!asc) && startKey.key.ts > pWindow->ekey)) {




2,147,483,647✔





2957



    startKey.key.ts = asc ? pWindow->skey : pWindow->ekey;




2,147,483,647✔





2958



    forward = false;




2,147,483,647✔





2959



  }










2960












2961



  code = doInitMemDataIter(pReader, &d, pBlockScanInfo, &startKey, pSnap->pMem, &pBlockScanInfo->iter, "mem");




2,147,483,647✔





2962



  TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





2963












2964



  code = doInitMemDataIter(pReader, &di, pBlockScanInfo, &startKey, pSnap->pIMem, &pBlockScanInfo->iiter, "imem");




2,147,483,647✔





2965



  TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





2966












2967



  code = loadMemTombData(&pBlockScanInfo->pMemDelData, d, di, pReader->info.verRange.maxVer);




2,147,483,647✔





2968



  TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





2969












2970



  if (forward) {




2,147,483,647✔





2971



    code = forwardDataIter(&startKey.key, pBlockScanInfo, pReader);




458,886,215✔





2972



    TSDB_CHECK_CODE(code, lino, _end);




458,890,871✔





2973



  }










2974












2975



  pBlockScanInfo->iterInit = true;




2,147,483,647✔





2976












2977



_end:




2,147,483,647✔





2978



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





2979



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







2980



  }










2981



  return code;




2,147,483,647✔





2982



}










2983












2984



static int32_t isValidFileBlockRow(SBlockData* pBlockData, int32_t rowIndex, STableBlockScanInfo* pBlockScanInfo,




2,147,483,647✔





2985



                                   bool asc, STsdbReaderInfo* pInfo, STsdbReader* pReader, bool* valid) {










2986



  int32_t code = TSDB_CODE_SUCCESS;




2,147,483,647✔





2987



  int32_t lino = 0;




2,147,483,647✔





2988












2989



  TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





2990



  TSDB_CHECK_NULL(valid, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





2991












2992



  *valid = false;




2,147,483,647✔





2993



  // it is an multi-table data block










2994



  if (pBlockData->aUid != NULL) {




2,147,483,647✔





2995



    uint64_t uid = pBlockData->aUid[rowIndex];




×







2996



    if (uid != pBlockScanInfo->uid) {  // move to next row




×







2997



      *valid = false;




×







2998



      goto _end;




×







2999



    }










3000



  }










3001












3002



  TSDB_CHECK_NULL(pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





3003



  // check for version and time range










3004



  int64_t ver = pBlockData->aVersion[rowIndex];




2,147,483,647✔





3005



  if (ver > pInfo->verRange.maxVer || ver < pInfo->verRange.minVer) {




2,147,483,647✔





3006



    *valid = false;




×







3007



    goto _end;




×







3008



  }










3009












3010



  int64_t ts = pBlockData->aTSKEY[rowIndex];




2,147,483,647✔





3011



  if (ts > pInfo->window.ekey || ts < pInfo->window.skey) {




2,147,483,647✔





3012



    *valid = false;




832,132,480✔





3013



    goto _end;




824,305,027✔





3014



  }










3015












3016



  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





3017



  if ((asc && (ts < pBlockScanInfo->lastProcKey.ts)) || ((!asc) && (ts > pBlockScanInfo->lastProcKey.ts))) {




2,147,483,647✔





3018



    *valid = false;




5,841✔





3019



    goto _end;




×







3020



  }










3021












3022



  if (ts == pBlockScanInfo->lastProcKey.ts) {  // todo opt perf




2,147,483,647✔





3023



    SRowKey nextRowKey;                        // lazy eval




×







3024



    tColRowGetKey(pBlockData, rowIndex, &nextRowKey);




×







3025



    if (pkCompEx(&pBlockScanInfo->lastProcKey, &nextRowKey) == 0) {




×







3026



      *valid = false;




×







3027



      goto _end;




×







3028



    }










3029



  }










3030












3031



  if (pBlockScanInfo->delSkyline != NULL && TARRAY_SIZE(pBlockScanInfo->delSkyline) > 0) {




2,147,483,647✔





3032



    bool dropped = false;




218,037,062✔





3033



    code = hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, ts, ver, pInfo->order,




218,037,062✔





3034



                          &pInfo->verRange, pReader->suppInfo.numOfPks > 0, &dropped);




218,037,062✔





3035



    TSDB_CHECK_CODE(code, lino, _end);




218,037,062✔





3036



    if (dropped) {




218,037,062✔





3037



      *valid = false;




83,387,292✔





3038



      goto _end;




83,387,292✔





3039



    }










3040



  }










3041












3042



  *valid = true;




2,147,483,647✔





3043












3044



_end:




2,147,483,647✔





3045



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





3046



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







3047



  }










3048



  return code;




2,147,483,647✔





3049



}










3050












3051



static void initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) {




2,147,483,647✔





3052



  int32_t              code = TSDB_CODE_SUCCESS;




2,147,483,647✔





3053



  int32_t              lino = 0;




2,147,483,647✔





3054



  int32_t              order = 0;




2,147,483,647✔





3055



  bool                 asc = false;




2,147,483,647✔





3056



  int64_t              st = 0;




2,147,483,647✔





3057



  SSttDataInfoForTable info = (SSttDataInfoForTable){0};




2,147,483,647✔





3058












3059



  TSDB_CHECK_NULL(pSttBlockReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





3060



  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





3061



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





3062












3063



  order = pReader->info.order;




2,147,483,647✔





3064



  asc = ASCENDING_TRAVERSE(order);




2,147,483,647✔





3065












3066



  // the stt block reader has been initialized for this table.










3067



  if (pSttBlockReader->uid == pScanInfo->uid) {




2,147,483,647✔





3068



    goto _end;




2,127,101,553✔





3069



  }










3070












3071



  if (pSttBlockReader->uid != 0) {




2,147,483,647✔





3072



    tMergeTreeClose(&pSttBlockReader->mergeTree);




2,000,145,123✔





3073



  }










3074












3075



  pSttBlockReader->uid = pScanInfo->uid;




2,147,483,647✔





3076












3077



  // second or third time init stt block reader










3078



  if (pScanInfo->cleanSttBlocks && (pReader->info.execMode == READER_EXEC_ROWS)) {




2,147,483,647✔





3079



    // only allowed to retrieve clean stt blocks for count once










3080



    if (pScanInfo->sttBlockReturned) {




×







3081



      pScanInfo->sttKeyInfo.status = STT_FILE_NO_DATA;




×







3082



      tsdbDebug("uid:%" PRIu64 " set no stt-file data after stt-block retrieved, %s", pScanInfo->uid, pReader->idStr);




×







3083



    }










3084



    goto _end;




×







3085



  }










3086












3087



  STimeWindow w = pSttBlockReader->window;




2,147,483,647✔





3088



  if (asc) {




2,147,483,647✔





3089



    w.skey = pScanInfo->sttKeyInfo.nextProcKey.ts;




2,147,483,647✔





3090



  } else {










3091



    w.ekey = pScanInfo->sttKeyInfo.nextProcKey.ts;




197,766,956✔





3092



  }










3093












3094



  // early filter for invalid time window










3095



  if (w.skey > w.ekey) {




2,147,483,647✔





3096



    pScanInfo->sttKeyInfo.status = STT_FILE_NO_DATA;




27,435✔





3097



    tsdbDebug("uid:%" PRIu64 " set no stt-file data, skey:%" PRId64 " > ekey:%" PRId64 " %s",




27,435✔





3098



              pScanInfo->uid, w.skey, w.ekey, pReader->idStr);










3099



    goto _end;




27,435✔





3100



  }










3101












3102



  st = taosGetTimestampUs();




2,147,483,647✔





3103



  tsdbDebug("init stt block reader, window:%" PRId64 "-%" PRId64 ", uid:%" PRIu64 ", %s", w.skey, w.ekey,




2,147,483,647✔





3104



            pScanInfo->uid, pReader->idStr);










3105












3106



  SMergeTreeConf conf = {




2,147,483,647✔





3107



      .uid = pScanInfo->uid,




2,147,483,647✔





3108



      .suid = pReader->info.suid,




2,147,483,647✔





3109



      .pTsdb = pReader->pTsdb,




2,147,483,647✔





3110



      .timewindow = w,










3111



      .verRange = pSttBlockReader->verRange,










3112



      .strictTimeRange = false,










3113



      .pSchema = pReader->info.pSchema,




2,147,483,647✔





3114



      .pCurrentFileset = pReader->status.pCurrentFileset,




2,147,483,647✔





3115



      .backward = (pSttBlockReader->order == TSDB_ORDER_DESC),




2,147,483,647✔





3116



      .pSttFileBlockIterArray = pReader->status.pLDataIterArray,




2,147,483,647✔





3117



      .pCols = pReader->suppInfo.colId,




2,147,483,647✔





3118



      .numOfCols = pReader->suppInfo.numOfCols,




2,147,483,647✔





3119



      .loadTombFn = loadSttTombDataForAll,










3120



      .pCurRowKey = &pScanInfo->sttKeyInfo.nextProcKey,




2,147,483,647✔





3121



      .pReader = pReader,










3122



      .idstr = pReader->idStr,




2,147,483,647✔





3123



      .rspRows = (pReader->info.execMode == READER_EXEC_ROWS),




2,147,483,647✔





3124



  };










3125












3126



  info.pKeyRangeList = taosArrayInit(4, sizeof(SSttKeyRange));




2,147,483,647✔





3127



  TSDB_CHECK_NULL(info.pKeyRangeList, code, lino, _end, terrno);




2,147,483,647✔





3128












3129



  code = tMergeTreeOpen2(&pSttBlockReader->mergeTree, &conf, &info);




2,147,483,647✔





3130



  TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





3131












3132



  code = initMemDataIterator(pScanInfo, pReader);




2,147,483,647✔





3133



  TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





3134












3135



  code = initDelSkylineIterator(pScanInfo, pReader->info.order, &pReader->cost);




2,147,483,647✔





3136



  TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





3137












3138



  if (conf.rspRows) {




2,147,483,647✔





3139



    pScanInfo->cleanSttBlocks = isCleanSttBlock(info.pKeyRangeList, &pReader->info.window, pScanInfo, order);




183,025,680✔





3140



    if (pScanInfo->cleanSttBlocks) {




183,018,505✔





3141



      pScanInfo->numOfRowsInStt = info.numOfRows;




39,260,131✔





3142












3143



      // calculate the time window for data in stt files










3144



      for (int32_t i = 0; i < taosArrayGetSize(info.pKeyRangeList); ++i) {




78,683,006✔





3145



        SSttKeyRange* pKeyRange = taosArrayGet(info.pKeyRangeList, i);




39,408,356✔





3146



        if (pKeyRange == NULL) {




39,414,197✔





3147



          continue;




×







3148



        }










3149












3150



        if (pkCompEx(&pScanInfo->sttRange.skey, &pKeyRange->skey) > 0) {




78,832,418✔





3151



          tRowKeyAssign(&pScanInfo->sttRange.skey, &pKeyRange->skey);




39,260,579✔





3152



        }










3153












3154



        if (pkCompEx(&pScanInfo->sttRange.ekey, &pKeyRange->ekey) < 0) {




78,822,505✔





3155



          tRowKeyAssign(&pScanInfo->sttRange.ekey, &pKeyRange->ekey);




39,417,565✔





3156



        }










3157



      }










3158












3159



      pScanInfo->sttKeyInfo.status = taosArrayGetSize(info.pKeyRangeList) ? STT_FILE_HAS_DATA : STT_FILE_NO_DATA;




39,274,727✔





3160












3161



      SRowKey* p = asc ? &pScanInfo->sttRange.skey : &pScanInfo->sttRange.ekey;




39,271,340✔





3162



      tRowKeyAssign(&pScanInfo->sttKeyInfo.nextProcKey, p);




39,274,153✔





3163



    } else {                                // not clean stt blocks










3164



      INIT_KEYRANGE(&pScanInfo->sttRange);  // reset the time window




143,706,961✔





3165



      code = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pReader->suppInfo.pkSrcSlot, &pReader->info.verRange);




143,759,586✔





3166



      if (code != TSDB_CODE_SUCCESS) {




143,729,567✔





3167



        pScanInfo->sttBlockReturned = false;




×







3168



        TSDB_CHECK_CODE(code, lino, _end);




×







3169



      }










3170



    }










3171



  } else {










3172



    pScanInfo->cleanSttBlocks = false;




2,147,483,647✔





3173



    INIT_KEYRANGE(&pScanInfo->sttRange);  // reset the time window




2,147,483,647✔





3174



    code = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pReader->suppInfo.pkSrcSlot, &pReader->info.verRange);




2,147,483,647✔





3175



    if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





3176



      pScanInfo->sttBlockReturned = false;




×







3177



      TSDB_CHECK_CODE(code, lino, _end);




×







3178



    }










3179



  }










3180












3181



  pScanInfo->sttBlockReturned = false;




2,147,483,647✔





3182



  int64_t el = taosGetTimestampUs() - st;




2,147,483,647✔





3183



  pReader->cost.initSttBlockReader += (el / 1000.0);




2,147,483,647✔





3184












3185



  tsdbDebug("init stt block reader completed, elapsed time:%" PRId64 "us %s", el, pReader->idStr);




2,147,483,647✔





3186












3187



_end:




2,147,483,647✔





3188



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





3189



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







3190



    if (pReader) {




×







3191



      pReader->code = code;




×







3192



    }










3193



  }










3194



  taosArrayDestroy(info.pKeyRangeList);




2,147,483,647✔





3195



}




2,147,483,647✔





3196












3197



static bool hasDataInSttBlock(STableBlockScanInfo* pInfo) { return pInfo->sttKeyInfo.status == STT_FILE_HAS_DATA; }




2,147,483,647✔





3198












3199



bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo) {




2,147,483,647✔





3200



  if ((pBlockData->nRow > 0) && (pBlockData->nRow != pDumpInfo->totalRows)) {




2,147,483,647✔





3201



    return false;  // this is an invalid result.




×







3202



  }










3203



  return pBlockData->nRow > 0 && (!pDumpInfo->allDumped);




2,147,483,647✔





3204



}










3205












3206



int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, SRowKey* pKey,




1,593,313,624✔





3207



                              STsdbReader* pReader) {










3208



  int32_t             code = TSDB_CODE_SUCCESS;




1,593,313,624✔





3209



  int32_t             lino = 0;




1,593,313,624✔





3210



  SRow*               pTSRow = NULL;




1,593,313,624✔





3211



  SRowMerger*         pMerger = NULL;




1,593,313,624✔





3212



  SFileBlockDumpInfo* pDumpInfo = NULL;




1,593,313,624✔





3213



  bool                copied = false;




1,593,313,624✔





3214












3215



  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,593,313,624✔





3216



  TSDB_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,593,313,624✔





3217



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,593,313,624✔





3218












3219



  pMerger = &pReader->status.merger;




1,593,313,624✔





3220



  pDumpInfo = &pReader->status.fBlockDumpInfo;




1,593,313,624✔





3221












3222



  code = tryCopyDistinctRowFromFileBlock(pReader, pBlockData, pKey, pDumpInfo, &copied);




1,593,313,624✔





3223



  TSDB_CHECK_CODE(code, lino, _end);




1,593,313,624✔





3224












3225



  // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized










3226



  code = initRowMergeIfNeeded(pReader, pBlockScanInfo->uid);




1,593,313,624✔





3227



  TSDB_CHECK_CODE(code, lino, _end);




1,593,313,624✔





3228












3229



  tRowKeyAssign(&pBlockScanInfo->lastProcKey, pKey);




1,593,313,624✔





3230












3231



  if (copied) {




1,593,313,624✔





3232



    goto _end;




1,592,276,473✔





3233



  }










3234












3235



  TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);




1,037,151✔





3236












3237



  code = tsdbRowMergerAdd(pMerger, &fRow, NULL);




1,037,151✔





3238



  TSDB_CHECK_CODE(code, lino, _end);




1,037,151✔





3239












3240



  code = doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader);




1,037,151✔





3241



  TSDB_CHECK_CODE(code, lino, _end);




1,037,151✔





3242












3243



  code = tsdbRowMergerGetRow(pMerger, &pTSRow);




1,037,151✔





3244



  TSDB_CHECK_CODE(code, lino, _end);




1,037,151✔





3245












3246



  code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo);




1,037,151✔





3247












3248



  taosMemoryFreeClear(pTSRow);




1,037,151✔





3249



  tsdbRowMergerClear(pMerger);




1,037,151✔





3250



  TSDB_CHECK_CODE(code, lino, _end);




1,037,151✔





3251












3252



_end:




1,593,313,624✔





3253



  if (code != TSDB_CODE_SUCCESS) {




1,593,313,624✔





3254



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







3255



  }










3256



  return code;




1,593,313,624✔





3257



}










3258












3259



int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) {




2,147,483,647✔





3260



  int32_t     code = TSDB_CODE_SUCCESS;




2,147,483,647✔





3261



  int32_t     lino = 0;




2,147,483,647✔





3262



  SRow*       pTSRow = NULL;




2,147,483,647✔





3263



  int32_t     pkSrcSlot = 0;




2,147,483,647✔





3264



  SRowMerger* pMerger = NULL;




2,147,483,647✔





3265



  bool        copied = false;




2,147,483,647✔





3266












3267



  TSDB_CHECK_NULL(pSttBlockReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





3268



  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





3269



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





3270












3271



  pkSrcSlot = pReader->suppInfo.pkSrcSlot;




2,147,483,647✔





3272



  pMerger = &pReader->status.merger;




2,147,483,647✔





3273












3274



  // let's record the last processed key










3275



  tRowKeyAssign(&pScanInfo->lastProcKey, getCurrentKeyInSttBlock(pSttBlockReader));




2,147,483,647✔





3276












3277



  TSDBROW* pRow = tMergeTreeGetRow(&pSttBlockReader->mergeTree);




2,147,483,647✔





3278



  TSDBROW  fRow = {.iRow = pRow->iRow, .type = TSDBROW_COL_FMT, .pBlockData = pRow->pBlockData};




2,147,483,647✔





3279












3280



  if (IS_VAR_DATA_TYPE(pScanInfo->lastProcKey.pks[0].type)) {




2,147,483,647✔





3281



    tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", ts:%" PRId64 " pk:%s %s", pRow->pBlockData, pRow->iRow,




768,977,050✔





3282



              pSttBlockReader->uid, fRow.pBlockData->aTSKEY[fRow.iRow], pScanInfo->lastProcKey.pks[0].pData,










3283



              pReader->idStr);










3284



  }










3285












3286



  code = tryCopyDistinctRowFromSttBlock(&fRow, pSttBlockReader, pScanInfo, &pScanInfo->lastProcKey, pReader, &copied);




2,147,483,647✔





3287



  TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





3288












3289



  if (copied) {




2,147,483,647✔





3290



    goto _end;




2,147,483,647✔





3291



  }










3292












3293



  code = tsdbRowMergerAdd(pMerger, &fRow, NULL);




1,638,364,380✔





3294



  TSDB_CHECK_CODE(code, lino, _end);




1,639,059,481✔





3295












3296



  TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree);




1,639,059,481✔





3297



  code = tsdbRowMergerAdd(pMerger, pRow1, NULL);




1,638,920,208✔





3298



  TSDB_CHECK_CODE(code, lino, _end);




1,639,171,139✔





3299












3300



  code = doMergeRowsInSttBlock(pSttBlockReader, pScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr);




1,639,171,139✔





3301



  TSDB_CHECK_CODE(code, lino, _end);




1,638,010,606✔





3302












3303



  code = tsdbRowMergerGetRow(pMerger, &pTSRow);




1,638,010,606✔





3304



  TSDB_CHECK_CODE(code, lino, _end);




1,637,380,012✔





3305












3306



  code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pScanInfo);




1,637,380,012✔





3307












3308



  taosMemoryFreeClear(pTSRow);




1,639,207,117✔





3309



  tsdbRowMergerClear(pMerger);




1,638,713,639✔





3310



  TSDB_CHECK_CODE(code, lino, _end);




1,639,156,037✔





3311












3312



_end:




2,147,483,647✔





3313



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





3314



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







3315



  }










3316



  return code;




2,147,483,647✔





3317



}










3318












3319



static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo,




2,147,483,647✔





3320



                                          SBlockData* pBlockData, SSttBlockReader* pSttBlockReader) {










3321



  int32_t             code = TSDB_CODE_SUCCESS;




2,147,483,647✔





3322



  int32_t             lino = 0;




2,147,483,647✔





3323



  SFileBlockDumpInfo* pDumpInfo = NULL;




2,147,483,647✔





3324



  TSDBROW*            pRow = NULL;




2,147,483,647✔





3325



  TSDBROW*            piRow = NULL;




2,147,483,647✔





3326



  SRowKey*            pKey = &(SRowKey){0};




2,147,483,647✔





3327












3328



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





3329



  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





3330



  TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





3331












3332



  pDumpInfo = &pReader->status.fBlockDumpInfo;




2,147,483,647✔





3333












3334



  if (hasDataInFileBlock(pBlockData, pDumpInfo)) {




2,147,483,647✔





3335



    tColRowGetKey(pBlockData, pDumpInfo->rowIndex, pKey);




2,147,483,647✔





3336



    code = primaryKeyDup(pKey);




2,147,483,647✔





3337



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





3338



    lino = __LINE__;




2,147,483,647✔





3339



  } else {










3340



    pKey = NULL;




2,147,483,647✔





3341



  }










3342












3343



  if (pBlockScanInfo->iter.hasVal) {




2,147,483,647✔





3344



    code = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader, &pRow);




2,147,483,647✔





3345



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





3346



  }










3347












3348



  if (pBlockScanInfo->iiter.hasVal) {




2,147,483,647✔





3349



    code = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader, &piRow);




2,147,483,647✔





3350



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





3351



  }










3352












3353



  if (pRow != NULL && piRow != NULL) {




2,147,483,647✔





3354



    // imem + mem + file + stt block










3355



    // two levels of mem-table does contain the valid rows










3356



    code = doMergeMultiLevelRows(pReader, pBlockScanInfo, pBlockData, pSttBlockReader);




295,687✔





3357



    TSDB_CHECK_CODE(code, lino, _end);




295,687✔





3358



  } else if (pBlockScanInfo->iiter.hasVal) {




2,147,483,647✔





3359



    // imem + file + stt block










3360



    code = doMergeBufAndFileRows(pReader, pBlockScanInfo, piRow, &pBlockScanInfo->iiter, pSttBlockReader);




2,147,483,647✔





3361



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





3362



  } else if (pBlockScanInfo->iter.hasVal) {




2,147,483,647✔





3363



    // mem + file + stt block










3364



    code = doMergeBufAndFileRows(pReader, pBlockScanInfo, pRow, &pBlockScanInfo->iter, pSttBlockReader);




2,147,483,647✔





3365



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





3366



  } else {










3367



    // files data blocks + stt block










3368



    code = mergeFileBlockAndSttBlock(pReader, pSttBlockReader, pKey, pBlockScanInfo, pBlockData);




2,147,483,647✔





3369



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





3370



  }










3371












3372



_end:




2,147,483,647✔





3373



  clearRowKey(pKey);




2,147,483,647✔





3374



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





3375



    tsdbError("%s %s failed at line %d since %s", pReader->idStr, __func__, lino, tstrerror(code));




×







3376



  }










3377



  return code;




2,147,483,647✔





3378



}










3379












3380



static int32_t loadNeighborIfOverlap(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pBlockScanInfo,




15,139,056✔





3381



                                     STsdbReader* pReader, bool* loadNeighbor) {










3382



  int32_t             code = TSDB_CODE_SUCCESS;




15,139,056✔





3383



  int32_t             lino = 0;




15,139,056✔





3384



  int32_t             order = 0;




15,139,056✔





3385



  SDataBlockIter*     pIter = NULL;




15,139,056✔





3386



  SBlockLoadSuppInfo* pSupInfo = NULL;




15,139,056✔





3387



  int32_t             step = 0;




15,139,056✔





3388



  int32_t             nextIndex = -1;




15,139,056✔





3389



  SBrinRecord         rec = {0};




15,139,056✔





3390



  bool                hasNeighbor = false;




15,139,056✔





3391












3392



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




15,139,056✔





3393



  TSDB_CHECK_NULL(loadNeighbor, code, lino, _end, TSDB_CODE_INVALID_PARA);




15,139,056✔





3394












3395



  order = pReader->info.order;




15,139,056✔





3396



  pIter = &pReader->status.blockIter;




15,139,056✔





3397



  pSupInfo = &pReader->suppInfo;




15,139,056✔





3398



  step = ASCENDING_TRAVERSE(order) ? 1 : -1;




15,139,056✔





3399












3400



  *loadNeighbor = false;




15,139,056✔





3401



  code = getNeighborBlockOfTable(pIter, pBlockInfo, pBlockScanInfo, &nextIndex, order, &rec, pSupInfo, &hasNeighbor);




15,139,056✔





3402



  TSDB_CHECK_CODE(code, lino, _end);




15,139,056✔





3403



  if (!hasNeighbor) {  // do nothing




15,139,056✔





3404



    goto _end;




5,486,882✔





3405



  }










3406












3407



  // load next block










3408



  if (overlapWithNeighborBlock2(pBlockInfo, &rec, order, pReader->suppInfo.pk.type, pReader->suppInfo.numOfPks)) {




9,652,174✔





3409



    SReaderStatus*  pStatus = &pReader->status;




8,220✔





3410



    SDataBlockIter* pBlockIter = &pStatus->blockIter;




8,220✔





3411












3412



    // 1. find the next neighbor block in the scan block list










3413



    STableDataBlockIdx* tableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, nextIndex);




8,220✔





3414



    TSDB_CHECK_NULL(tableDataBlockIdx, code, lino, _end, TSDB_CODE_INVALID_PARA);




8,220✔





3415












3416



    // 2. remove it from the scan block list










3417



    int32_t neighborIndex = tableDataBlockIdx->globalIndex;




8,220✔





3418



    code = setFileBlockActiveInBlockIter(pReader, pBlockIter, neighborIndex, step);




8,220✔





3419



    TSDB_CHECK_CODE(code, lino, _end);




8,220✔





3420












3421



    // 3. load the neighbor block, and set it to be the currently accessed file data block










3422



    code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pBlockInfo->uid);




8,220✔





3423



    TSDB_CHECK_CODE(code, lino, _end);




8,220✔





3424












3425



    // 4. check the data values










3426



    code = initBlockDumpInfo(pReader, pBlockIter);




8,220✔





3427



    TSDB_CHECK_CODE(code, lino, _end);




8,220✔





3428



    *loadNeighbor = true;




8,220✔





3429



  }










3430












3431



_end:




15,139,056✔





3432



  if (code != TSDB_CODE_SUCCESS) {




15,139,056✔





3433



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







3434



  }










3435



  return code;




15,139,056✔





3436



}










3437












3438



void updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInfo* pBlockScanInfo) {




2,147,483,647✔





3439



  SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock;




2,147,483,647✔





3440












3441



  pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0;




2,147,483,647✔





3442



  pResBlock->info.dataLoad = 1;




2,147,483,647✔





3443



  pResBlock->info.version = pReader->info.verRange.maxVer;




2,147,483,647✔





3444












3445



  int32_t code = blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotId[0]);




2,147,483,647✔





3446



  code = blockDataUpdatePkRange(pResBlock, pReader->suppInfo.pkDstSlot, ASCENDING_TRAVERSE(pReader->info.order));




2,147,483,647✔





3447



  setComposedBlockFlag(pReader, true);




2,147,483,647✔





3448












3449



  pReader->cost.composedBlocks += 1;




2,147,483,647✔





3450



  pReader->cost.buildComposedBlockTime += el;




2,147,483,647✔





3451



}




2,147,483,647✔





3452












3453



static int32_t buildComposedDataBlock(STsdbReader* pReader) {




16,633,047✔





3454



  int32_t              code = TSDB_CODE_SUCCESS;




16,633,047✔





3455



  int32_t              lino = 0;




16,633,047✔





3456



  bool                 asc = false;




16,633,047✔





3457



  int64_t              st = 0;




16,633,047✔





3458



  double               el = 0;




16,633,047✔





3459



  int32_t              step = 0;




16,633,047✔





3460



  SSDataBlock*         pResBlock = NULL;




16,633,047✔





3461



  SFileDataBlockInfo*  pBlockInfo = NULL;




16,633,047✔





3462



  SSttBlockReader*     pSttBlockReader = NULL;




16,633,047✔





3463



  SFileBlockDumpInfo*  pDumpInfo = NULL;




16,633,047✔





3464



  STableBlockScanInfo* pBlockScanInfo = NULL;




16,633,047✔





3465



  TSDBKEY              keyInBuf;




16,633,047✔





3466












3467



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




16,633,047✔





3468












3469



  asc = ASCENDING_TRAVERSE(pReader->info.order);




16,633,047✔





3470



  st = taosGetTimestampUs();




16,633,047✔





3471



  step = asc ? 1 : -1;




16,633,047✔





3472



  pResBlock = pReader->resBlockInfo.pResBlock;




16,633,047✔





3473



  pSttBlockReader = pReader->status.fileIter.pSttBlockReader;




16,632,283✔





3474



  pDumpInfo = &pReader->status.fBlockDumpInfo;




16,633,047✔





3475












3476



  code = getCurrentBlockInfo(&pReader->status.blockIter, &pBlockInfo, pReader->idStr);




16,633,047✔





3477



  if (code != TSDB_CODE_SUCCESS) {




16,632,283✔





3478



    return 0;




×







3479



  }










3480












3481



  if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) {




16,632,283✔





3482



    setBlockAllDumped(pDumpInfo, pBlockInfo->lastKey, pReader->info.order);




×







3483



    return code;




×







3484



  }










3485












3486



  code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr);




16,633,047✔





3487



  TSDB_CHECK_CODE(code, lino, _end);




16,633,047✔





3488












3489



  code = getCurrentKeyInBuf(pBlockScanInfo, pReader, &keyInBuf);




16,633,047✔





3490



  TSDB_CHECK_CODE(code, lino, _end);




16,632,283✔





3491












3492



  // it is a clean block, load it directly










3493



  int64_t cap = pReader->resBlockInfo.capacity;




16,632,283✔





3494



  bool    isClean = false;




16,632,283✔





3495



  code = isCleanFileDataBlock(pReader, pBlockInfo, pBlockScanInfo, keyInBuf, &isClean);




16,632,283✔





3496



  TSDB_CHECK_CODE(code, lino, _end);




16,632,283✔





3497



  bool directCopy = isClean && (pBlockInfo->numRow <= cap) && (pBlockScanInfo->sttKeyInfo.status == STT_FILE_NO_DATA) &&




18,126,274✔





3498



                    ((asc && ((pBlockInfo->lastKey < keyInBuf.ts) || (keyInBuf.ts == INT64_MIN))) ||




988,279✔





3499



                     (!asc && (pBlockInfo->lastKey > keyInBuf.ts)));




505,712✔





3500



  if (directCopy) {




16,632,283✔





3501



    code = copyBlockDataToSDataBlock(pReader, &pBlockScanInfo->lastProcKey);




1,493,991✔





3502



    TSDB_CHECK_CODE(code, lino, _end);




1,493,991✔





3503



    goto _end;




1,493,991✔





3504



  }










3505












3506



  SBlockData* pBlockData = &pReader->status.fileBlockData;




15,138,292✔





3507



  initSttBlockReader(pSttBlockReader, pBlockScanInfo, pReader);




15,138,292✔





3508



  code = pReader->code;




15,138,292✔





3509



  TSDB_CHECK_CODE(code, lino, _end);




329,701✔





3510












3511



  while (1) {




2,147,483,647✔





3512



    bool hasBlockData = false;




2,147,483,647✔





3513



    {










3514



      while (pBlockData->nRow > 0 && pBlockData->uid == pBlockScanInfo->uid) {




2,147,483,647✔





3515



        // find the first qualified row in data block










3516



        bool valid = false;




2,147,483,647✔





3517



        code =










3518



            isValidFileBlockRow(pBlockData, pDumpInfo->rowIndex, pBlockScanInfo, asc, &pReader->info, pReader, &valid);




2,147,483,647✔





3519



        TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





3520



        if (valid) {




2,147,483,647✔





3521



          hasBlockData = true;




2,147,483,647✔





3522



          break;




2,147,483,647✔





3523



        }










3524












3525



        pDumpInfo->rowIndex += step;




907,665,698✔





3526












3527



        if (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0) {




907,701,510✔





3528



          // NOTE: get the new block info










3529



          code = getCurrentBlockInfo(&pReader->status.blockIter, &pBlockInfo, pReader->idStr);




1,051,949✔





3530



          TSDB_CHECK_CODE(code, lino, _end);




1,051,703✔





3531












3532



          // continue check for the next file block if the last ts in the current block










3533



          // is overlapped with the next neighbor block










3534



          bool loadNeighbor = false;




1,051,703✔





3535



          code = loadNeighborIfOverlap(pBlockInfo, pBlockScanInfo, pReader, &loadNeighbor);




1,051,703✔





3536



          if ((!loadNeighbor) || (code != 0)) {




1,051,703✔





3537



            lino = __LINE__;




1,051,703✔





3538



            setBlockAllDumped(pDumpInfo, pBlockInfo->lastKey, pReader->info.order);




1,051,703✔





3539



            break;




1,051,703✔





3540



          }










3541



        }










3542



      }










3543



    }










3544












3545



    // no data in last block and block, no need to proceed.










3546



    if (hasBlockData == false) {




2,147,483,647✔





3547



      break;




1,051,703✔





3548



    }










3549












3550



    code = buildComposedDataBlockImpl(pReader, pBlockScanInfo, pBlockData, pSttBlockReader);




2,147,483,647✔





3551



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





3552












3553



    // currently loaded file data block is consumed










3554



    if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) {




2,147,483,647✔





3555



      setBlockAllDumped(pDumpInfo, pBlockInfo->lastKey, pReader->info.order);




13,233,541✔





3556



      break;




14,079,133✔





3557



    }










3558












3559



    if (pResBlock->info.rows >= pReader->resBlockInfo.capacity) {




2,147,483,647✔





3560



      break;




8,220✔





3561



    }










3562



  }










3563












3564



_end:




3,234,309✔





3565



  if (pReader) {




16,633,047✔





3566



    el = (taosGetTimestampUs() - st) / 1000.0;




16,633,047✔





3567



    updateComposedBlockInfo(pReader, el, pBlockScanInfo);




16,633,047✔





3568



  }










3569












3570



  if (pResBlock && pResBlock->info.rows > 0) {




16,633,047✔





3571



    tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%" PRId64




16,045,336✔





3572



              ", elapsed time:%.2f ms %s",










3573



              pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey,










3574



              pResBlock->info.rows, el, pReader->idStr);










3575



  }










3576












3577



  if (code != TSDB_CODE_SUCCESS) {




16,633,047✔





3578



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







3579



  }










3580



  return code;




16,633,047✔





3581



}










3582












3583



void setComposedBlockFlag(STsdbReader* pReader, bool composed) { pReader->status.composedDataBlock = composed; }




2,147,483,647✔





3584












3585



int32_t getInitialDelIndex(const SArray* pDelSkyline, int32_t order) {




51,037,209✔





3586



  if (pDelSkyline == NULL) {




51,037,209✔





3587



    return 0;




×







3588



  }










3589












3590



  return ASCENDING_TRAVERSE(order) ? 0 : taosArrayGetSize(pDelSkyline) - 1;




51,037,209✔





3591



}










3592












3593



int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SReadCostSummary* pCost) {




2,147,483,647✔





3594



  int32_t code = TSDB_CODE_SUCCESS;




2,147,483,647✔





3595



  int32_t lino = 0;




2,147,483,647✔





3596



  int32_t newDelDataInFile = 0;




2,147,483,647✔





3597



  int64_t st = 0;




2,147,483,647✔





3598



  SArray* pSource = NULL;




2,147,483,647✔





3599












3600



  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





3601












3602



  newDelDataInFile = taosArrayGetSize(pBlockScanInfo->pFileDelData);




2,147,483,647✔





3603



  if (newDelDataInFile == 0 &&




2,147,483,647✔





3604



      ((pBlockScanInfo->delSkyline != NULL) || (TARRAY_SIZE(pBlockScanInfo->pMemDelData) == 0))) {




2,147,483,647✔





3605



    goto _end;




2,147,483,647✔





3606



  }










3607












3608



  st = taosGetTimestampUs();




51,043,126✔





3609












3610



  if (pBlockScanInfo->delSkyline != NULL) {




51,043,126✔





3611



    taosArrayClear(pBlockScanInfo->delSkyline);




7,423,065✔





3612



  } else {










3613



    pBlockScanInfo->delSkyline = taosArrayInit(4, sizeof(TSDBKEY));




43,615,380✔





3614



    TSDB_CHECK_NULL(pBlockScanInfo->delSkyline, code, lino, _end, terrno);




43,615,881✔





3615



  }










3616












3617



  pSource = pBlockScanInfo->pFileDelData;




51,041,740✔





3618



  if (pSource == NULL) {




51,040,655✔





3619



    pSource = pBlockScanInfo->pMemDelData;




11,911,435✔





3620



  } else {










3621



    const void* p1 = taosArrayAddAll(pSource, pBlockScanInfo->pMemDelData);




39,129,220✔





3622



    TSDB_CHECK_NULL(p1, code, lino, _end, terrno);




39,130,305✔





3623



  }










3624












3625



  code = tsdbBuildDeleteSkyline(pSource, 0, taosArrayGetSize(pSource) - 1, pBlockScanInfo->delSkyline);




51,043,174✔





3626












3627



  taosArrayClear(pBlockScanInfo->pFileDelData);




51,038,454✔





3628



  int32_t index = getInitialDelIndex(pBlockScanInfo->delSkyline, order);




51,041,322✔





3629












3630



  pBlockScanInfo->iter.index = index;




51,038,294✔





3631



  pBlockScanInfo->iiter.index = index;




51,038,813✔





3632



  pBlockScanInfo->fileDelIndex = index;




51,037,331✔





3633



  pBlockScanInfo->sttBlockDelIndex = index;




51,037,209✔





3634



  TSDB_CHECK_CODE(code, lino, _end);




51,039,580✔





3635












3636



  TSDB_CHECK_NULL(pCost, code, lino, _end, TSDB_CODE_INVALID_PARA);




51,039,580✔





3637



  double el = taosGetTimestampUs() - st;




51,039,797✔





3638



  pCost->createSkylineIterTime = el / 1000.0;




51,039,797✔





3639












3640



_end:




2,147,483,647✔





3641



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





3642



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







3643



  }










3644



  return code;




2,147,483,647✔





3645



}










3646












3647



int32_t getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader, TSDBKEY* pkey) {




566,896,631✔





3648



  int32_t  code = TSDB_CODE_SUCCESS;




566,896,631✔





3649



  int32_t  lino = 0;




566,896,631✔





3650



  bool     asc = false;




566,896,631✔





3651



  bool     hasKey = false;




566,896,631✔





3652



  bool     hasIKey = false;




566,896,631✔





3653



  TSDBROW* pRow = NULL;




566,896,631✔





3654



  TSDBROW* pIRow = NULL;




566,907,516✔





3655



  TSDBKEY  key = {.ts = TSKEY_INITIAL_VAL};




566,921,457✔





3656



  TSDBKEY  ikey = {.ts = TSKEY_INITIAL_VAL};




566,921,457✔





3657












3658



  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




566,921,457✔





3659



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




566,921,457✔





3660



  TSDB_CHECK_NULL(pkey, code, lino, _end, TSDB_CODE_INVALID_PARA);




566,921,457✔





3661












3662



  asc = ASCENDING_TRAVERSE(pReader->info.order);




566,921,457✔





3663












3664



  code = getValidMemRow(&pScanInfo->iter, pScanInfo->delSkyline, pReader, &pRow);




566,907,381✔





3665



  TSDB_CHECK_CODE(code, lino, _end);




566,837,824✔





3666



  if (pRow != NULL) {




566,837,824✔





3667



    hasKey = true;




38,919,689✔





3668



    key = TSDBROW_KEY(pRow);




38,919,689✔





3669



  }










3670












3671



  code = getValidMemRow(&pScanInfo->iiter, pScanInfo->delSkyline, pReader, &pIRow);




566,837,060✔





3672



  TSDB_CHECK_CODE(code, lino, _end);




566,810,551✔





3673



  if (pIRow != NULL) {




566,810,551✔





3674



    hasIKey = true;




8,570,088✔





3675



    ikey = TSDBROW_KEY(pIRow);




8,570,088✔





3676



  }










3677












3678



  if (hasKey) {




566,810,551✔





3679



    if (hasIKey) {  // has data in mem & imem




38,918,925✔





3680



      if (asc) {




538,692✔





3681



        *pkey = key.ts <= ikey.ts ? key : ikey;




538,692✔





3682



      } else {










3683



        *pkey = key.ts <= ikey.ts ? ikey : key;




×







3684



      }










3685



    } else {  // no data in imem










3686



      *pkey = key;




38,380,233✔





3687



    }










3688



  } else {










3689



    // no data in mem & imem, return the initial value










3690



    // only imem has data, return ikey










3691



    *pkey = ikey;




527,891,626✔





3692



  }










3693












3694



_end:




566,794,925✔





3695



  if (code != TSDB_CODE_SUCCESS) {




566,934,704✔





3696



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







3697



  }










3698



  return code;




566,934,704✔





3699



}










3700












3701



static void prepareDurationForNextFileSet(STsdbReader* pReader) {




139,833,662✔





3702



  if (pReader->status.bProcMemFirstFileset) {




139,833,662✔





3703



    pReader->status.prevFilesetStartKey = INT64_MIN;




40,036,921✔





3704



    pReader->status.prevFilesetEndKey = INT64_MAX;




40,068,095✔





3705



    pReader->status.bProcMemFirstFileset = false;




40,062,026✔





3706



  }










3707












3708



  int32_t     fid = pReader->status.pCurrentFileset->fid;




139,791,834✔





3709



  STimeWindow winFid = {0};




139,849,519✔





3710



  tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &winFid.skey, &winFid.ekey);




139,885,635✔





3711












3712



  if (ASCENDING_TRAVERSE(pReader->info.order)) {




139,850,957✔





3713



    pReader->status.bProcMemPreFileset = !(pReader->status.memTableMaxKey < pReader->status.prevFilesetStartKey ||




130,277,395✔





3714



                                           (winFid.skey - 1) < pReader->status.memTableMinKey);




38,033,420✔





3715



  } else {










3716



    pReader->status.bProcMemPreFileset = !(pReader->status.memTableMaxKey < (winFid.ekey + 1) ||




47,852,586✔





3717



                                           pReader->status.prevFilesetEndKey < pReader->status.memTableMinKey);




219,575✔





3718



  }










3719












3720



  if (pReader->status.bProcMemPreFileset) {




139,761,917✔





3721



    tsdbDebug("will start pre-fileset %d buffer processing. %s", fid, pReader->idStr);




2,281,604✔





3722



    pReader->status.procMemUidList.tableUidList = pReader->status.uidList.tableUidList;




2,281,604✔





3723



    resetPreFilesetMemTableListIndex(&pReader->status);




2,281,604✔





3724



  }










3725












3726



  if (!pReader->status.bProcMemPreFileset) {




139,827,147✔





3727



    if (pReader->notifyFn) {




137,541,930✔





3728



      STsdReaderNotifyInfo info = {0};




136,417,142✔





3729



      info.duration.filesetId = fid;




136,424,760✔





3730



      pReader->notifyFn(TSD_READER_NOTIFY_DURATION_START, &info, pReader->notifyParam);




136,424,760✔





3731



      tsdbDebug("new duration %d start notification when no buffer preceeding fileset, %s", fid, pReader->idStr);




136,492,701✔





3732



    }










3733



  }










3734












3735



  pReader->status.prevFilesetStartKey = winFid.skey;




139,909,642✔





3736



  pReader->status.prevFilesetEndKey = winFid.ekey;




139,900,191✔





3737



}




139,896,814✔





3738












3739



static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SArray* pTableList) {




2,147,483,647✔





3740



  int32_t        code = TSDB_CODE_SUCCESS;




2,147,483,647✔





3741



  int32_t        lino = 0;




2,147,483,647✔





3742



  SReaderStatus* pStatus = NULL;




2,147,483,647✔





3743



  SArray*        pIndexList = NULL;




2,147,483,647✔





3744



  size_t         numOfTables = 0;




2,147,483,647✔





3745












3746



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





3747



  TSDB_CHECK_NULL(pBlockNum, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





3748












3749



  pStatus = &pReader->status;




2,147,483,647✔





3750



  pBlockNum->numOfBlocks = 0;




2,147,483,647✔





3751



  pBlockNum->numOfSttFiles = 0;




2,147,483,647✔





3752












3753



  numOfTables = tSimpleHashGetSize(pReader->status.pTableMap);




2,147,483,647✔





3754



  pIndexList = taosArrayInit(numOfTables, sizeof(SBrinBlk));




2,147,483,647✔





3755



  TSDB_CHECK_NULL(pIndexList, code, lino, _end, terrno);




2,147,483,647✔





3756












3757



  while (1) {




2,068,888✔





3758



    // only check here, since the iterate data in memory is very fast.










3759



    if (pReader->code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





3760



      tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", tstrerror(pReader->code), pReader->idStr);




×







3761



      code = pReader->code;




×







3762



      TSDB_CHECK_CODE(code, lino, _end);




×







3763



    }










3764












3765



    bool hasNext = false;




2,147,483,647✔





3766



    code = filesetIteratorNext(&pStatus->fileIter, pReader, &hasNext);




2,147,483,647✔





3767



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





3768












3769



    if (!hasNext) {  // no data files on disk




2,147,483,647✔





3770



      break;




420,414,670✔





3771



    }










3772












3773



    taosArrayClear(pIndexList);




2,147,483,647✔





3774



    code = doLoadBlockIndex(pReader, pReader->pFileReader, pIndexList);




2,147,483,647✔





3775



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





3776












3777



    if (taosArrayGetSize(pIndexList) > 0 || pReader->status.pCurrentFileset->lvlArr->size > 0) {




2,147,483,647✔





3778



      code = loadFileBlockBrinInfo(pReader, pIndexList, pBlockNum, pTableList);




2,147,483,647✔





3779



      TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





3780












3781



      if (pBlockNum->numOfBlocks + pBlockNum->numOfSttFiles > 0) {




2,147,483,647✔





3782



        if (pReader->bFilesetDelimited) {




2,147,483,647✔





3783



          prepareDurationForNextFileSet(pReader);




139,844,934✔





3784



        }










3785



        break;




2,147,483,647✔





3786



      }










3787



    }










3788












3789



    // no blocks in current file, try next files










3790



  }










3791












3792



  code = loadDataFileTombDataForAll(pReader);




2,147,483,647✔





3793



  TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





3794












3795



_end:




2,147,483,647✔





3796



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





3797



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







3798



  }










3799



  if (pIndexList) {




2,147,483,647✔





3800



    taosArrayDestroy(pIndexList);




2,147,483,647✔





3801



  }










3802



  return code;




2,147,483,647✔





3803



}










3804












3805



// pTableIter can be NULL, no need to handle the return value










3806



static int32_t resetTableListIndex(SReaderStatus* pStatus, const char* id) {




2,147,483,647✔





3807



  int32_t        code = TSDB_CODE_SUCCESS;




2,147,483,647✔





3808



  int32_t        lino = 0;




2,147,483,647✔





3809



  STableUidList* pList = NULL;




2,147,483,647✔





3810












3811



  TSDB_CHECK_NULL(pStatus, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





3812












3813



  pList = &pStatus->uidList;




2,147,483,647✔





3814












3815



  pList->currentIndex = 0;




2,147,483,647✔





3816



  uint64_t uid = pList->tableUidList[0];




2,147,483,647✔





3817



  pStatus->pTableIter = tSimpleHashGet(pStatus->pTableMap, &uid, sizeof(uid));




2,147,483,647✔





3818



  if (pStatus->pTableIter == NULL) {




2,147,483,647✔





3819



    tsdbError("%s failed to load tableBlockScanInfo for uid:%" PRId64 ", code: internal error", id, uid);




×







3820



    TSDB_CHECK_NULL(pStatus->pTableIter, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);




×







3821



  }










3822












3823



_end:




2,147,483,647✔





3824



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





3825



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







3826



  }










3827



  return code;




2,147,483,647✔





3828



}










3829












3830



static void resetPreFilesetMemTableListIndex(SReaderStatus* pStatus) {




2,281,604✔





3831



  STableUidList* pList = &pStatus->procMemUidList;




2,281,604✔





3832












3833



  pList->currentIndex = 0;




2,281,604✔





3834



  uint64_t uid = pList->tableUidList[0];




2,281,604✔





3835



  pStatus->pProcMemTableIter = tSimpleHashGet(pStatus->pTableMap, &uid, sizeof(uid));




2,281,604✔





3836



}




2,281,604✔





3837












3838



static bool moveToNextTable(STableUidList* pOrderedCheckInfo, SReaderStatus* pStatus) {




2,147,483,647✔





3839



  pOrderedCheckInfo->currentIndex += 1;




2,147,483,647✔





3840



  if (pOrderedCheckInfo->currentIndex >= tSimpleHashGetSize(pStatus->pTableMap)) {




2,147,483,647✔





3841



    pStatus->pTableIter = NULL;




2,147,483,647✔





3842



    return false;




2,147,483,647✔





3843



  }










3844












3845



  uint64_t uid = pOrderedCheckInfo->tableUidList[pOrderedCheckInfo->currentIndex];




2,147,483,647✔





3846



  pStatus->pTableIter = tSimpleHashGet(pStatus->pTableMap, &uid, sizeof(uid));




2,147,483,647✔





3847



  return (pStatus->pTableIter != NULL);




2,147,483,647✔





3848



}










3849












3850



static bool moveToNextTableForPreFileSetMem(SReaderStatus* pStatus) {




3,312,780✔





3851



  STableUidList* pUidList = &pStatus->procMemUidList;




3,312,780✔





3852



  pUidList->currentIndex += 1;




3,312,780✔





3853



  if (pUidList->currentIndex >= tSimpleHashGetSize(pStatus->pTableMap)) {




3,312,780✔





3854



    pStatus->pProcMemTableIter = NULL;




2,281,604✔





3855



    return false;




2,281,604✔





3856



  }










3857












3858



  uint64_t uid = pUidList->tableUidList[pUidList->currentIndex];




1,031,176✔





3859



  pStatus->pProcMemTableIter = tSimpleHashGet(pStatus->pTableMap, &uid, sizeof(uid));




1,031,176✔





3860



  return (pStatus->pProcMemTableIter != NULL);




1,031,176✔





3861



}










3862












3863



static int32_t buildCleanBlockFromSttFiles(STsdbReader* pReader, STableBlockScanInfo* pScanInfo) {




39,270,909✔





3864



  int32_t          code = TSDB_CODE_SUCCESS;




39,270,909✔





3865



  int32_t          lino = 0;




39,270,909✔





3866



  SReaderStatus*   pStatus = NULL;




39,270,909✔





3867



  SSttBlockReader* pSttBlockReader = NULL;




39,270,909✔





3868



  SSDataBlock*     pResBlock = NULL;




39,270,909✔





3869



  SDataBlockInfo*  pInfo = NULL;




39,270,909✔





3870



  bool             asc = false;




39,270,909✔





3871












3872



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




39,270,909✔





3873



  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




39,270,909✔





3874












3875



  pStatus = &pReader->status;




39,270,909✔





3876



  pSttBlockReader = pStatus->fileIter.pSttBlockReader;




39,264,274✔





3877



  pResBlock = pReader->resBlockInfo.pResBlock;




39,273,475✔





3878












3879



  asc = ASCENDING_TRAVERSE(pReader->info.order);




39,264,560✔





3880



  code = blockDataEnsureCapacity(pResBlock, pScanInfo->numOfRowsInStt);




39,261,190✔





3881



  TSDB_CHECK_CODE(code, lino, _end);




39,275,265✔





3882












3883



  pInfo = &pResBlock->info;




39,275,265✔





3884



  pInfo->rows = pScanInfo->numOfRowsInStt;




39,277,700✔





3885



  pInfo->id.uid = pScanInfo->uid;




39,272,535✔





3886



  pInfo->dataLoad = 1;




39,273,219✔





3887



  pInfo->window.skey = pScanInfo->sttRange.skey.ts;




39,263,916✔





3888



  pInfo->window.ekey = pScanInfo->sttRange.ekey.ts;




39,270,732✔





3889












3890



  setComposedBlockFlag(pReader, true);




39,269,032✔





3891












3892



  pScanInfo->sttKeyInfo.nextProcKey.ts = asc ? pScanInfo->sttRange.ekey.ts + 1 : pScanInfo->sttRange.skey.ts - 1;




39,262,538✔





3893



  pScanInfo->sttKeyInfo.status = STT_FILE_NO_DATA;




39,237,358✔





3894












3895



  if (asc) {




39,254,586✔





3896



    tRowKeyAssign(&pScanInfo->lastProcKey, &pScanInfo->sttRange.ekey);




39,254,586✔





3897



  } else {










3898



    tRowKeyAssign(&pScanInfo->lastProcKey, &pScanInfo->sttRange.skey);




×







3899



  }










3900












3901



  pScanInfo->sttBlockReturned = true;




39,271,760✔





3902



  pSttBlockReader->mergeTree.pIter = NULL;




39,270,334✔





3903












3904



  tsdbDebug("%p uid:%" PRId64 " return clean stt block as one, brange:%" PRId64 "-%" PRId64 " rows:%" PRId64 " %s",




39,247,547✔





3905



            pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey,










3906



            pResBlock->info.rows, pReader->idStr);










3907












3908



_end:




39,240,584✔





3909



  if (code != TSDB_CODE_SUCCESS) {




39,258,600✔





3910



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







3911



  }










3912



  return code;




39,256,720✔





3913



}










3914












3915



static int32_t buildCleanBlockFromDataFiles(STsdbReader* pReader, STableBlockScanInfo* pScanInfo,




532,658,655✔





3916



                                            SFileDataBlockInfo* pBlockInfo, int32_t blockIndex) {










3917



  int32_t         code = TSDB_CODE_SUCCESS;




532,658,655✔





3918



  int32_t         lino = 0;




532,658,655✔





3919



  SReaderStatus*  pStatus = NULL;




532,658,655✔





3920



  SDataBlockInfo* pInfo = NULL;




532,658,655✔





3921



  bool            asc = false;




532,658,655✔





3922












3923



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




532,658,655✔





3924



  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




532,658,655✔





3925



  TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




532,658,655✔





3926












3927



  // whole data block is required, return it directly










3928



  pStatus = &pReader->status;




532,658,655✔





3929



  pInfo = &pReader->resBlockInfo.pResBlock->info;




532,656,315✔





3930



  asc = ASCENDING_TRAVERSE(pReader->info.order);




532,714,869✔





3931












3932



  pInfo->rows = pBlockInfo->numRow;




532,782,445✔





3933



  pInfo->id.uid = pScanInfo->uid;




532,752,497✔





3934



  pInfo->dataLoad = 0;




532,812,712✔





3935



  pInfo->version = pReader->info.verRange.maxVer;




532,788,916✔





3936



  pInfo->window = (STimeWindow){.skey = pBlockInfo->firstKey, .ekey = pBlockInfo->lastKey};




532,713,589✔





3937












3938



  if (pReader->suppInfo.numOfPks > 0) {




532,764,780✔





3939



    if (IS_NUMERIC_TYPE(pReader->suppInfo.pk.type)) {




4,260✔





3940



      VALUE_SET_TRIVIAL_DATUM(pInfo->pks, pBlockInfo->firstPk.val);




×







3941



      VALUE_SET_TRIVIAL_DATUM(pInfo->pks + 1, pBlockInfo->lastPk.val);




×







3942



    } else {










3943



      (void)memcpy(pInfo->pks[0].pData, varDataVal(pBlockInfo->firstPk.pData), varDataLen(pBlockInfo->firstPk.pData));




4,260✔





3944



      (void)memcpy(pInfo->pks[1].pData, varDataVal(pBlockInfo->lastPk.pData), varDataLen(pBlockInfo->lastPk.pData));




4,260✔





3945












3946



      pInfo->pks[0].nData = varDataLen(pBlockInfo->firstPk.pData);




4,260✔





3947



      pInfo->pks[1].nData = varDataLen(pBlockInfo->lastPk.pData);




4,260✔





3948



    }










3949



  }










3950












3951



  // update the last key for the corresponding table










3952



  setComposedBlockFlag(pReader, false);




532,776,239✔





3953



  setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->lastKey, pReader->info.order);




532,733,029✔





3954



  code = updateLastKeyInfo(&pScanInfo->lastProcKey, pBlockInfo, pInfo, pReader->suppInfo.numOfPks, asc);




532,739,852✔





3955



  TSDB_CHECK_CODE(code, lino, _end);




532,685,950✔





3956












3957



  tsdbDebug("%p uid:%" PRIu64




532,685,950✔





3958



            " clean file block retrieved from file, global index:%d, "










3959



            "table index:%d, rows:%d, brange:%" PRId64 "-%" PRId64 ", %s",










3960



            pReader, pScanInfo->uid, blockIndex, pBlockInfo->tbBlockIdx, pBlockInfo->numRow, pBlockInfo->firstKey,










3961



            pBlockInfo->lastKey, pReader->idStr);










3962












3963



_end:




514,476,344✔





3964



  if (code != TSDB_CODE_SUCCESS) {




532,669,938✔





3965



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







3966



  }










3967



  return code;




532,607,140✔





3968



}










3969












3970



static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) {




2,147,483,647✔





3971



  int32_t              code = TSDB_CODE_SUCCESS;




2,147,483,647✔





3972



  int32_t              lino = 0;




2,147,483,647✔





3973



  SReaderStatus*       pStatus = NULL;




2,147,483,647✔





3974



  SSttBlockReader*     pSttBlockReader = NULL;




2,147,483,647✔





3975



  STableUidList*       pUidList = NULL;




2,147,483,647✔





3976



  SSDataBlock*         pResBlock = NULL;




2,147,483,647✔





3977



  STableBlockScanInfo* pScanInfo = NULL;




2,147,483,647✔





3978












3979



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





3980












3981



  pStatus = &pReader->status;




2,147,483,647✔





3982



  pSttBlockReader = pStatus->fileIter.pSttBlockReader;




2,147,483,647✔





3983



  pUidList = &pStatus->uidList;




2,147,483,647✔





3984












3985



  if (tSimpleHashGetSize(pStatus->pTableMap) == 0) {




2,147,483,647✔





3986



    goto _end;




×







3987



  }










3988












3989



  pResBlock = pReader->resBlockInfo.pResBlock;




2,147,483,647✔





3990












3991



  while (1) {




1,741,782,792✔





3992



    if (pReader->code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





3993



      tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", tstrerror(pReader->code), pReader->idStr);




×







3994



      code = pReader->code;




×







3995



      TSDB_CHECK_CODE(code, lino, _end);




×







3996



    }










3997












3998



    // load the last data block of current table










3999



    if (pStatus->pTableIter == NULL) {




2,147,483,647✔





4000



      tsdbError("table Iter is null, invalid pScanInfo, try next table %s", pReader->idStr);




×







4001



      bool hasNexTable = moveToNextTable(pUidList, pStatus);




×







4002



      if (!hasNexTable) {




×







4003



        break;




×







4004



      }










4005












4006



      continue;




×







4007



    } else {










4008



      pScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter;




2,147,483,647✔





4009



    }










4010












4011



    if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pScanInfo->uid, sizeof(pScanInfo->uid))) {




2,147,483,647✔





4012



      // reset the index in last block when handing a new file










4013



      bool hasNexTable = moveToNextTable(pUidList, pStatus);




19,595,605✔





4014



      if (!hasNexTable) {




19,594,722✔





4015



        break;




1,675,788✔





4016



      }










4017












4018



      continue;




17,918,934✔





4019



    }










4020












4021



    initSttBlockReader(pSttBlockReader, pScanInfo, pReader);




2,147,483,647✔





4022



    code = pReader->code;




2,147,483,647✔





4023



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





4024












4025



    if (!hasDataInSttBlock(pScanInfo)) {




2,147,483,647✔





4026



      bool hasNexTable = moveToNextTable(pUidList, pStatus);




2,147,483,647✔





4027



      if (!hasNexTable) {




2,147,483,647✔





4028



        break;




2,147,483,647✔





4029



      }










4030












4031



      continue;




1,723,863,858✔





4032



    }










4033












4034



    // if only require the total rows, no need to load data from stt file if it is clean stt blocks










4035



    if (pReader->info.execMode == READER_EXEC_ROWS && pScanInfo->cleanSttBlocks) {




2,069,443,722✔





4036



      code = buildCleanBlockFromSttFiles(pReader, pScanInfo);




39,259,848✔





4037



      TSDB_CHECK_CODE(code, lino, _end);




39,256,855✔





4038



      break;




39,256,855✔





4039



    }










4040












4041



    int64_t st = taosGetTimestampUs();




2,030,090,532✔





4042



    while (1) {










4043



      // no data in stt block and block, no need to proceed.










4044



      if (!hasDataInSttBlock(pScanInfo)) {




2,147,483,647✔





4045



        break;




1,954,205,629✔





4046



      }










4047












4048



      code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pSttBlockReader);




2,147,483,647✔





4049



      TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





4050












4051



      if (pResBlock->info.rows >= pReader->resBlockInfo.capacity) {




2,147,483,647✔





4052



        break;




76,496,344✔





4053



      }










4054



    }










4055












4056



    double el = (taosGetTimestampUs() - st) / 1000.0;




2,030,554,114✔





4057



    updateComposedBlockInfo(pReader, el, pScanInfo);




2,030,554,114✔





4058












4059



    if (pResBlock->info.rows > 0) {




2,028,819,138✔





4060



      tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%" PRId64




2,029,107,172✔





4061



                ", elapsed time:%.2f ms %s",










4062



                pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey,










4063



                pResBlock->info.rows, el, pReader->idStr);










4064



      break;




2,028,981,716✔





4065



    }










4066












4067



    // current table is exhausted, let's try next table










4068



    bool hasNexTable = moveToNextTable(pUidList, pStatus);




×







4069



    if (!hasNexTable) {




×







4070



      break;




×







4071



    }










4072



  }










4073












4074



_end:




2,147,483,647✔





4075



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





4076



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







4077



  }










4078



  return code;




2,147,483,647✔





4079



}










4080












4081



// current active data block not overlap with the stt-files/stt-blocks










4082



static bool notOverlapWithFiles(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pScanInfo, bool asc) {




533,523,504✔





4083



  if ((!hasDataInSttBlock(pScanInfo)) || (pScanInfo->cleanSttBlocks == true)) {




533,523,504✔





4084



    return true;




532,665,618✔





4085



  } else {










4086



    int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey.ts;




890,760✔





4087



    return (asc && pBlockInfo->lastKey < keyInStt) || (!asc && pBlockInfo->firstKey > keyInStt);




890,760✔





4088



  }










4089



}










4090












4091



static int32_t doBuildDataBlock(STsdbReader* pReader) {




553,724,889✔





4092



  int32_t              code = TSDB_CODE_SUCCESS;




553,724,889✔





4093



  int32_t              lino = 0;




553,724,889✔





4094



  SReaderStatus*       pStatus = NULL;




553,724,889✔





4095



  SDataBlockIter*      pBlockIter = NULL;




553,724,889✔





4096



  STableBlockScanInfo* pScanInfo = NULL;




553,724,889✔





4097



  SFileDataBlockInfo*  pBlockInfo = NULL;




553,755,741✔





4098



  SSttBlockReader*     pSttBlockReader = NULL;




553,797,847✔





4099



  TSDBKEY              keyInBuf;




553,797,847✔





4100



  bool                 asc = false;




553,791,845✔





4101












4102



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




553,791,845✔





4103












4104



  pStatus = &pReader->status;




553,791,845✔





4105



  pBlockIter = &pStatus->blockIter;




553,786,813✔





4106



  pSttBlockReader = pReader->status.fileIter.pSttBlockReader;




553,825,770✔





4107



  asc = ASCENDING_TRAVERSE(pReader->info.order);




553,826,134✔





4108












4109



  code = getCurrentBlockInfo(pBlockIter, &pBlockInfo, pReader->idStr);




553,761,247✔





4110



  TSDB_CHECK_CODE(code, lino, _end);




553,832,548✔





4111












4112



  if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) {




553,832,548✔





4113



    setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->lastKey, pReader->info.order);




3,493,936✔





4114



    goto _end;




3,493,936✔





4115



  }










4116












4117



  code = pReader->code;




550,319,372✔





4118



  TSDB_CHECK_CODE(code, lino, _end);




550,299,224✔





4119












4120



  code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pScanInfo, pReader->idStr);




550,299,224✔





4121



  TSDB_CHECK_CODE(code, lino, _end);




550,287,104✔





4122












4123



  if (pScanInfo->sttKeyInfo.status == STT_FILE_READER_UNINIT) {




550,287,104✔





4124



    initSttBlockReader(pSttBlockReader, pScanInfo, pReader);




338,879,897✔





4125



    code = pReader->code;




338,852,286✔





4126



    TSDB_CHECK_CODE(code, lino, _end);




338,858,234✔





4127



  }










4128












4129



  code = getCurrentKeyInBuf(pScanInfo, pReader, &keyInBuf);




550,226,137✔





4130



  TSDB_CHECK_CODE(code, lino, _end);




550,291,693✔





4131



  bool load = false;




550,291,693✔





4132



  code = fileBlockShouldLoad(pReader, pBlockInfo, pScanInfo, keyInBuf, &load);




550,256,908✔





4133



  TSDB_CHECK_CODE(code, lino, _end);




550,076,818✔





4134



  if (load) {




550,076,818✔





4135



    code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid);




16,627,441✔





4136



    TSDB_CHECK_CODE(code, lino, _end);




16,628,625✔





4137












4138



    // build composed data block










4139



    code = buildComposedDataBlock(pReader);




16,628,625✔





4140



    TSDB_CHECK_CODE(code, lino, _end);




16,628,625✔





4141



  } else if (bufferDataInFileBlockGap(keyInBuf, pBlockInfo, pScanInfo, pReader->info.order)) {




533,449,377✔





4142



    // data in memory that are earlier than current file block and stt blocks










4143



    // rows in buffer should be less than the file block in asc, greater than file block in desc










4144



    int64_t endKey = getBoarderKeyInFiles(pBlockInfo, pScanInfo, pReader->info.order);




5,447✔





4145



    code = buildDataBlockFromBuf(pReader, pScanInfo, endKey);




5,447✔





4146



    TSDB_CHECK_CODE(code, lino, _end);




5,447✔





4147



  } else {










4148



    if (notOverlapWithFiles(pBlockInfo, pScanInfo, asc)) {




533,492,332✔





4149



      int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey.ts;




532,691,652✔





4150












4151



      if ((!hasDataInSttBlock(pScanInfo)) || (asc && pBlockInfo->lastKey < keyInStt) ||




532,741,156✔





4152



          (!asc && pBlockInfo->firstKey > keyInStt)) {




5,045✔





4153



        // the stt blocks may located in the gap of different data block, but the whole sttRange may overlap with the










4154



        // data block, so the overlap check is invalid actually.










4155



        code = buildCleanBlockFromDataFiles(pReader, pScanInfo, pBlockInfo, pBlockIter->index);




532,766,346✔





4156



        TSDB_CHECK_CODE(code, lino, _end);




532,615,826✔





4157



      } else {  // clean stt block










4158



        TSDB_CHECK_CONDITION((pReader->info.execMode == READER_EXEC_ROWS) && (pSttBlockReader->mergeTree.pIter == NULL),




5,045✔





4159



                             code, lino, _end, TSDB_CODE_INTERNAL_ERROR);










4160



        code = buildCleanBlockFromSttFiles(pReader, pScanInfo);




5,045✔





4161



        TSDB_CHECK_CODE(code, lino, _end);




5,045✔





4162



      }










4163



    } else {










4164



      SBlockData* pBData = &pReader->status.fileBlockData;




801,556✔





4165



      tBlockDataReset(pBData);




877,737✔





4166












4167



      SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock;




877,737✔





4168












4169



      tsdbDebug("load data in stt block firstly %s", pReader->idStr);




877,737✔





4170



      int64_t st = taosGetTimestampUs();




877,737✔





4171












4172



      // let's load data from stt files, make sure clear the cleanStt block flag before load the data from stt files










4173



      initSttBlockReader(pSttBlockReader, pScanInfo, pReader);




877,737✔





4174



      code = pReader->code;




877,737✔





4175



      TSDB_CHECK_CODE(code, lino, _end);




877,737✔





4176












4177



      // no data in stt block, no need to proceed.










4178



      while (hasDataInSttBlock(pScanInfo)) {




12,650,733✔





4179



        TSDB_CHECK_CONDITION(pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA, code, lino, _end,




11,772,996✔





4180



                             TSDB_CODE_INTERNAL_ERROR);










4181












4182



        code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pSttBlockReader);




11,772,996✔





4183



        TSDB_CHECK_CODE(code, lino, _end);




11,772,996✔





4184












4185



        if (pResBlock->info.rows >= pReader->resBlockInfo.capacity) {




11,772,996✔





4186



          break;




×







4187



        }










4188












4189



        // data in stt now overlaps with current active file data block, need to composed with file data block.










4190



        SRowKey* pSttKey = getCurrentKeyInSttBlock(pSttBlockReader);




11,772,996✔





4191



        if ((pSttKey->ts >= pBlockInfo->firstKey && asc) || (pSttKey->ts <= pBlockInfo->lastKey && (!asc))) {




11,772,996✔





4192



          tsdbDebug("%p lastKeyInStt:%" PRId64 ", overlap with file block, brange:%" PRId64 "-%" PRId64 " %s", pReader,




×







4193



                    pSttKey->ts, pBlockInfo->firstKey, pBlockInfo->lastKey, pReader->idStr);










4194



          break;




×







4195



        }










4196



      }










4197












4198



      double el = (taosGetTimestampUs() - st) / 1000.0;




877,737✔





4199



      updateComposedBlockInfo(pReader, el, pScanInfo);




877,737✔





4200












4201



      if (pResBlock->info.rows > 0) {




877,737✔





4202



        tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%" PRId64




877,737✔





4203



                  ", elapsed time:%.2f ms %s",










4204



                  pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey,










4205



                  pResBlock->info.rows, el, pReader->idStr);










4206



      }










4207



    }










4208



  }










4209












4210



  code = pReader->code;




550,132,680✔





4211



  TSDB_CHECK_CODE(code, lino, _end);




550,252,635✔





4212












4213



_end:




553,746,571✔





4214



  if (code != TSDB_CODE_SUCCESS) {




553,787,962✔





4215



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







4216



  }










4217



  return code;




553,787,962✔





4218



}










4219












4220



static int32_t buildBlockFromBufferSeqForPreFileset(STsdbReader* pReader, int64_t endKey) {




4,406,937✔





4221



  int32_t        code = TSDB_CODE_SUCCESS;




4,406,937✔





4222



  int32_t        lino = 0;




4,406,937✔





4223



  SReaderStatus* pStatus = NULL;




4,406,937✔





4224












4225



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




4,406,937✔





4226












4227



  pStatus = &pReader->status;




4,406,937✔





4228












4229



  tsdbDebug("seq load data blocks from cache that preceeds fileset %d, %s", pReader->status.pCurrentFileset->fid,




4,406,937✔





4230



            pReader->idStr);










4231












4232



  while (1) {




1,031,176✔





4233



    if (pReader->code != TSDB_CODE_SUCCESS) {




5,438,113✔





4234



      tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", tstrerror(pReader->code), pReader->idStr);




×







4235



      code = pReader->code;




×







4236



      TSDB_CHECK_CODE(code, lino, _end);




×







4237



    }










4238












4239



    STableBlockScanInfo** pBlockScanInfo = pStatus->pProcMemTableIter;




5,438,113✔





4240



    if (pReader->pIgnoreTables &&




10,876,226✔





4241



        taosHashGet(*pReader->pIgnoreTables, &(*pBlockScanInfo)->uid, sizeof((*pBlockScanInfo)->uid))) {




5,438,113✔





4242



      bool hasNexTable = moveToNextTableForPreFileSetMem(pStatus);




658,440✔





4243



      if (!hasNexTable) {




658,440✔





4244



        break;




263,376✔





4245



      }










4246



      continue;




395,064✔





4247



    }










4248












4249



    code = initMemDataIterator(*pBlockScanInfo, pReader);




4,779,673✔





4250



    TSDB_CHECK_CODE(code, lino, _end);




4,779,673✔





4251












4252



    code = initDelSkylineIterator(*pBlockScanInfo, pReader->info.order, &pReader->cost);




4,779,673✔





4253



    TSDB_CHECK_CODE(code, lino, _end);




4,779,673✔





4254












4255



    code = buildDataBlockFromBuf(pReader, *pBlockScanInfo, endKey);




4,779,673✔





4256



    TSDB_CHECK_CODE(code, lino, _end);




4,779,673✔





4257












4258



    if (pReader->resBlockInfo.pResBlock->info.rows > 0) {




4,779,673✔





4259



      break;




2,125,333✔





4260



    }










4261












4262



    // current table is exhausted, let's try next table










4263



    bool hasNexTable = moveToNextTableForPreFileSetMem(pStatus);




2,654,340✔





4264



    if (!hasNexTable) {




2,654,340✔





4265



      break;




2,018,228✔





4266



    }










4267



  }










4268












4269



_end:




4,406,937✔





4270



  if (code != TSDB_CODE_SUCCESS) {




4,406,937✔





4271



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







4272



  }










4273



  return code;




4,406,937✔





4274



}










4275












4276



static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader, int64_t endKey) {




2,147,483,647✔





4277



  int32_t        code = TSDB_CODE_SUCCESS;




2,147,483,647✔





4278



  int32_t        lino = 0;




2,147,483,647✔





4279



  SReaderStatus* pStatus = NULL;




2,147,483,647✔





4280



  STableUidList* pUidList = NULL;




2,147,483,647✔





4281












4282



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





4283












4284



  pStatus = &pReader->status;




2,147,483,647✔





4285



  pUidList = &pStatus->uidList;




2,147,483,647✔





4286












4287



  tsdbDebug("seq load data blocks from cache, %s", pReader->idStr);




2,147,483,647✔





4288












4289



  while (1) {




2,147,483,647✔





4290



    if (pReader->code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





4291



      tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", tstrerror(pReader->code), pReader->idStr);




×







4292



      code = pReader->code;




×







4293



      TSDB_CHECK_CODE(code, lino, _end);




×







4294



    }










4295












4296



    STableBlockScanInfo** pBlockScanInfo = pStatus->pTableIter;




2,147,483,647✔





4297



    if (pBlockScanInfo == NULL || *pBlockScanInfo == NULL) {




2,147,483,647✔





4298



      break;










4299



    }










4300












4301



    if (pReader->pIgnoreTables &&




2,147,483,647✔





4302



        taosHashGet(*pReader->pIgnoreTables, &(*pBlockScanInfo)->uid, sizeof((*pBlockScanInfo)->uid))) {




2,147,483,647✔





4303



      bool hasNexTable = moveToNextTable(pUidList, pStatus);




141,606,347✔





4304



      if (!hasNexTable) {




141,605,871✔





4305



        break;




2,957,180✔





4306



      }










4307



      continue;




138,648,691✔





4308



    }










4309












4310



    code = initMemDataIterator(*pBlockScanInfo, pReader);




2,147,483,647✔





4311



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





4312












4313



    code = initDelSkylineIterator(*pBlockScanInfo, pReader->info.order, &pReader->cost);




2,147,483,647✔





4314



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





4315












4316



    code = buildDataBlockFromBuf(pReader, *pBlockScanInfo, endKey);




2,147,483,647✔





4317



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





4318












4319



    if (pReader->resBlockInfo.pResBlock->info.rows > 0) {




2,147,483,647✔





4320



      break;




2,147,483,647✔





4321



    }










4322












4323



    // current table is exhausted, let's try next table










4324



    bool hasNexTable = moveToNextTable(pUidList, pStatus);




2,147,483,647✔





4325



    if (!hasNexTable) {




2,147,483,647✔





4326



      break;




1,662,003,846✔





4327



    }










4328



  }










4329












4330



_end:




193,745✔





4331



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





4332



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







4333



  }










4334



  return code;




2,147,483,647✔





4335



}










4336












4337



// set the correct start position in case of the first/last file block, according to the query time window










4338



static int32_t initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) {




2,147,483,647✔





4339



  int32_t             code = TSDB_CODE_SUCCESS;




2,147,483,647✔





4340



  int32_t             lino = 0;




2,147,483,647✔





4341



  SFileDataBlockInfo* pBlockInfo = NULL;




2,147,483,647✔





4342



  SReaderStatus*      pStatus = NULL;




2,147,483,647✔





4343



  SFileBlockDumpInfo* pDumpInfo = NULL;




2,147,483,647✔





4344












4345



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





4346












4347



  pStatus = &pReader->status;




2,147,483,647✔





4348



  pDumpInfo = &pStatus->fBlockDumpInfo;




2,147,483,647✔





4349












4350



  code = getCurrentBlockInfo(pBlockIter, &pBlockInfo, pReader->idStr);




2,147,483,647✔





4351



  if (code == TSDB_CODE_SUCCESS) {




2,147,483,647✔





4352



    pDumpInfo->totalRows = pBlockInfo->numRow;




553,129,108✔





4353



    pDumpInfo->rowIndex = ASCENDING_TRAVERSE(pReader->info.order) ? 0 : pBlockInfo->numRow - 1;




553,168,451✔





4354



  } else {










4355



    pDumpInfo->totalRows = 0;




2,147,483,647✔





4356



    pDumpInfo->rowIndex = 0;




2,147,483,647✔





4357



    code = TSDB_CODE_SUCCESS;




2,147,483,647✔





4358



  }










4359












4360



  pDumpInfo->allDumped = false;




2,147,483,647✔





4361












4362



_end:




2,147,483,647✔





4363



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





4364



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







4365



  }










4366



  return code;




2,147,483,647✔





4367



}










4368












4369



static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBlockIter) {




2,147,483,647✔





4370



  int32_t      code = TSDB_CODE_SUCCESS;




2,147,483,647✔





4371



  int32_t      lino = 0;




2,147,483,647✔





4372



  SBlockNumber num = {0};




2,147,483,647✔





4373



  SArray*      pTableList = NULL;




2,147,483,647✔





4374












4375



  pTableList = taosArrayInit(40, POINTER_BYTES);




2,147,483,647✔





4376



  TSDB_CHECK_NULL(pTableList, code, lino, _end, terrno);




2,147,483,647✔





4377












4378



  code = moveToNextFile(pReader, &num, pTableList);




2,147,483,647✔





4379



  TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





4380












4381



  // all data files are consumed, try data in buffer










4382



  if (num.numOfBlocks + num.numOfSttFiles == 0) {




2,147,483,647✔





4383



    pReader->status.loadFromFile = false;




420,389,667✔





4384



    goto _end;




420,430,113✔





4385



  }










4386












4387



  // initialize the block iterator for a new fileset










4388



  if (num.numOfBlocks > 0) {




2,147,483,647✔





4389



    code = initBlockIterator(pReader, pBlockIter, num.numOfBlocks, pTableList);




145,829,299✔





4390



    TSDB_CHECK_CODE(code, lino, _end);




145,896,381✔





4391



  } else {  // no block data, only last block exists










4392



    tBlockDataReset(&pReader->status.fileBlockData);




2,147,483,647✔





4393



    code = resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo), pReader->idStr);




2,147,483,647✔





4394



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





4395












4396



    code = resetTableListIndex(&pReader->status, pReader->idStr);




2,147,483,647✔





4397



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





4398



  }










4399












4400



  code = initBlockDumpInfo(pReader, pBlockIter);




2,147,483,647✔





4401



  TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





4402












4403



_end:




2,147,483,647✔





4404



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





4405



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







4406



  }










4407



  if (pTableList) {




2,147,483,647✔





4408



    taosArrayDestroy(pTableList);




2,147,483,647✔





4409



  }










4410



  return code;




2,147,483,647✔





4411



}










4412












4413



static bool fileBlockPartiallyRead(SFileBlockDumpInfo* pDumpInfo, bool asc) {




573,558,623✔





4414



  return (!pDumpInfo->allDumped) &&




595,217,323✔





4415



         ((pDumpInfo->rowIndex > 0 && asc) || (pDumpInfo->rowIndex < (pDumpInfo->totalRows - 1) && (!asc)));




21,670,586✔





4416



}










4417












4418



typedef enum {










4419



  TSDB_READ_RETURN = 0x1,










4420



  TSDB_READ_CONTINUE = 0x2,










4421



} ERetrieveType;










4422












4423



static int32_t doReadDataFromSttFiles(STsdbReader* pReader, ERetrieveType* pReturnType) {




2,147,483,647✔





4424



  int32_t         code = TSDB_CODE_SUCCESS;




2,147,483,647✔





4425



  int32_t         lino = 0;




2,147,483,647✔





4426



  SSDataBlock*    pResBlock = NULL;




2,147,483,647✔





4427



  SDataBlockIter* pBlockIter = NULL;




2,147,483,647✔





4428












4429



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





4430



  TSDB_CHECK_NULL(pReturnType, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





4431












4432



  pResBlock = pReader->resBlockInfo.pResBlock;




2,147,483,647✔





4433



  pBlockIter = &pReader->status.blockIter;




2,147,483,647✔





4434












4435



  *pReturnType = TSDB_READ_RETURN;




2,147,483,647✔





4436












4437



  tsdbDebug("seq load data blocks from stt files %s", pReader->idStr);




2,147,483,647✔





4438












4439



  while (1) {










4440



    code = doLoadSttBlockSequentially(pReader);




2,147,483,647✔





4441



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





4442












4443



    if (pResBlock->info.rows > 0) {




2,147,483,647✔





4444



      goto _end;




2,068,912,464✔





4445



    }










4446












4447



    // all data blocks are checked in this stt file, now let's try the next file set










4448



    TSDB_CHECK_CONDITION(pReader->status.pTableIter == NULL, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);




2,147,483,647✔





4449












4450



    code = initForFirstBlockInFile(pReader, pBlockIter);




2,147,483,647✔





4451



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





4452












4453



    // all the data files are completely checked










4454



    if (pReader->status.loadFromFile == false) {




2,147,483,647✔





4455



      goto _end;




418,272,117✔





4456



    }










4457












4458



    if (pReader->status.bProcMemPreFileset) {




2,147,483,647✔





4459



      code = buildFromPreFilesetBuffer(pReader);




2,245,516✔





4460



      TSDB_CHECK_CODE(code, lino, _end);




2,245,516✔





4461



      if (pResBlock->info.rows > 0) {




2,245,516✔





4462



        pReader->status.processingMemPreFileSet = true;




1,482,709✔





4463



        goto _end;




1,482,709✔





4464



      }










4465



    }










4466












4467



    if (pBlockIter->numOfBlocks > 0) {  // there are data blocks existed.




2,147,483,647✔





4468



      *pReturnType = TSDB_READ_CONTINUE;




124,913,741✔





4469



      goto _end;




124,907,042✔





4470



    } else {  // all blocks in data file are checked, let's check the data in stt-files










4471



      code = resetTableListIndex(&pReader->status, pReader->idStr);




2,147,483,647✔





4472



      TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





4473



    }










4474



  }










4475












4476



_end:




2,147,483,647✔





4477



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





4478



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







4479



  }










4480



  return code;




2,147,483,647✔





4481



}










4482












4483



static int32_t buildBlockFromFiles(STsdbReader* pReader) {




2,147,483,647✔





4484



  int32_t             code = TSDB_CODE_SUCCESS;




2,147,483,647✔





4485



  int32_t             lino = 0;




2,147,483,647✔





4486



  SDataBlockIter*     pBlockIter = NULL;




2,147,483,647✔





4487



  SSDataBlock*        pResBlock = NULL;




2,147,483,647✔





4488



  SFileBlockDumpInfo* pDumpInfo = NULL;




2,147,483,647✔





4489



  SBlockData*         pBlockData = NULL;




2,147,483,647✔





4490



  const char*         id = NULL;




2,147,483,647✔





4491



  bool                asc = false;




2,147,483,647✔





4492












4493



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





4494












4495



  asc = ASCENDING_TRAVERSE(pReader->info.order);




2,147,483,647✔





4496












4497



  pBlockIter = &pReader->status.blockIter;




2,147,483,647✔





4498



  pResBlock = pReader->resBlockInfo.pResBlock;




2,147,483,647✔





4499



  pDumpInfo = &pReader->status.fBlockDumpInfo;




2,147,483,647✔





4500



  pBlockData = &pReader->status.fileBlockData;




2,147,483,647✔





4501



  id = pReader->idStr;




2,147,483,647✔





4502












4503



  if (pBlockIter->numOfBlocks == 0) {




2,147,483,647✔





4504



    // let's try to extract data from stt files.










4505



    ERetrieveType type = 0;




2,147,483,647✔





4506



    code = doReadDataFromSttFiles(pReader, &type);




2,147,483,647✔





4507



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





4508



    if (type == TSDB_READ_RETURN) {




2,147,483,647✔





4509



      goto _end;




2,147,483,647✔





4510



    }










4511












4512



    code = doBuildDataBlock(pReader);




427,384✔





4513



    TSDB_CHECK_CODE(code, lino, _end);




427,384✔





4514



    if (pResBlock->info.rows > 0) {




427,384✔





4515



      goto _end;




423,274✔





4516



    }










4517



  }










4518












4519



  while (1) {










4520



    if (fileBlockPartiallyRead(pDumpInfo, asc)) {  // file data block is partially loaded




573,560,492✔





4521



      code = buildComposedDataBlock(pReader);




4,422✔





4522



      TSDB_CHECK_CODE(code, lino, _end);




4,422✔





4523



    } else {










4524



      // current block are exhausted, try the next file block










4525



      if (pDumpInfo->allDumped) {




573,581,987✔





4526



        // try next data block in current file










4527



        bool hasNext = blockIteratorNext(&pReader->status.blockIter);




551,914,672✔





4528



        if (hasNext) {  // check for the next block in the block accessed order list




551,937,342✔





4529



          code = initBlockDumpInfo(pReader, pBlockIter);




407,249,046✔





4530



          TSDB_CHECK_CODE(code, lino, _end);




407,250,289✔





4531



        } else {










4532



          // all data blocks in files are checked, let's check the data in last files.










4533



          // data blocks in current file are exhausted, let's try the next file now










4534



          if (pBlockData->uid != 0) {




144,688,296✔





4535



            tBlockDataClear(pBlockData);




139,440,976✔





4536



          }










4537












4538



          tBlockDataReset(pBlockData);




144,710,774✔





4539



          code = resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo), id);




144,699,428✔





4540



          TSDB_CHECK_CODE(code, lino, _end);




144,707,255✔





4541












4542



          code = resetTableListIndex(&pReader->status, id);




144,707,255✔





4543



          TSDB_CHECK_CODE(code, lino, _end);




144,699,038✔





4544












4545



          ERetrieveType type = 0;




144,699,038✔





4546



          code = doReadDataFromSttFiles(pReader, &type);




144,711,135✔





4547



          TSDB_CHECK_CODE(code, lino, _end);




144,628,760✔





4548



          if (type == TSDB_READ_RETURN) {




144,628,760✔





4549



            break;




20,212,753✔





4550



          }










4551



        }










4552



      }










4553












4554



      code = doBuildDataBlock(pReader);




553,731,607✔





4555



      TSDB_CHECK_CODE(code, lino, _end);




553,323,299✔





4556



    }










4557












4558



    if (pResBlock->info.rows > 0) {




553,327,721✔





4559



      break;




549,273,280✔





4560



    }










4561



  }










4562












4563



_end:




2,147,483,647✔





4564



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





4565



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







4566



  }










4567



  return code;




2,147,483,647✔





4568



}










4569












4570



static void getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idStr,




1,891,273,352✔





4571



                                int8_t* pLevel, STsdb** pTsdb) {










4572



  if (pTsdb == NULL) {




1,891,273,352✔





4573



    return;




×







4574



  }










4575












4576



  *pTsdb = NULL;




1,891,273,352✔





4577



#if 0










4578



  if (VND_IS_RSMA(pVnode) && !pCond->skipRollup) {










4579



    int8_t  level = 0;










4580



    int8_t  precision = pVnode->config.tsdbCfg.precision;










4581



    int64_t now = taosGetTimestamp(precision);










4582



    int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI)   ? 1L










4583



                                             : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000L










4584



                                                                                        : 1000000L);










4585












4586



    for (int32_t i = 0; i < TSDB_RETENTION_MAX; ++i) {










4587



      SRetention* pRetention = retentions + level;










4588



      if (pRetention->keep <= 0) {










4589



        if (level > 0) {










4590



          --level;










4591



        }










4592



        break;










4593



      }










4594



      if ((now - pRetention->keep) <= (pCond->twindows.skey + offset)) {










4595



        break;










4596



      }










4597



      ++level;










4598



    }










4599












4600



    const char* str = (idStr != NULL) ? idStr : "";










4601












4602



    if (level == TSDB_RETENTION_L0) {










4603



      *pLevel = TSDB_RETENTION_L0;










4604



      tsdbDebug("vgId:%d, rsma level %d is selected to query %s", TD_VID(pVnode), TSDB_RETENTION_L0, str);










4605



      *pTsdb = VND_RSMA0(pVnode);










4606



      return;










4607



    } else if (level == TSDB_RETENTION_L1) {










4608



      *pLevel = TSDB_RETENTION_L1;










4609



      tsdbDebug("vgId:%d, rsma level %d is selected to query %s", TD_VID(pVnode), TSDB_RETENTION_L1, str);










4610



      *pTsdb = VND_RSMA1(pVnode);










4611



      return;










4612



    } else {










4613



      *pLevel = TSDB_RETENTION_L2;










4614



      tsdbDebug("vgId:%d, rsma level %d is selected to query %s", TD_VID(pVnode), TSDB_RETENTION_L2, str);










4615



      *pTsdb = VND_RSMA2(pVnode);










4616



      return;










4617



    }










4618



  }










4619



#endif










4620



  *pTsdb = VND_TSDB(pVnode);




1,891,700,459✔





4621



}










4622












4623



SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond) {




1,891,350,035✔





4624



  int64_t startVer = (pCond->startVersion == -1) ? 0 : pCond->startVersion;




1,891,350,035✔





4625












4626



  int64_t endVer = 0;




1,891,337,934✔





4627



  if (pCond->endVersion == -1) {




1,891,337,934✔





4628



    // user not specified end version, set current maximum version of vnode as the endVersion










4629



    endVer = pVnode->state.applied;




1,776,123,900✔





4630



  } else {










4631



    endVer = (pCond->endVersion > pVnode->state.applied) ? pVnode->state.applied : pCond->endVersion;




115,084,923✔





4632



  }










4633












4634



  return (SVersionRange){.minVer = startVer, .maxVer = endVer};




1,891,028,170✔





4635



}










4636












4637



static int32_t reverseSearchStartPos(const SArray* pDelList, int32_t index, int64_t key, bool asc, int32_t* start) {




18,629,573✔





4638



  int32_t code = TSDB_CODE_SUCCESS;




18,629,573✔





4639



  int32_t lino = 0;




18,629,573✔





4640



  size_t  num = 0;




18,629,573✔





4641












4642



  num = taosArrayGetSize(pDelList);




18,629,573✔





4643



  *start = index;




18,631,282✔





4644












4645



  if (asc) {




18,631,282✔





4646



    if (*start >= num - 1) {




11,323,870✔





4647



      *start = num - 1;




498,372✔





4648



    }










4649












4650



    TSDBKEY* p = taosArrayGet(pDelList, *start);




11,322,161✔





4651



    TSDB_CHECK_NULL(p, code, lino, _end, TSDB_CODE_INVALID_PARA);




11,323,870✔





4652












4653



    while (p->ts >= key && *start > 0) {




11,859,934✔





4654



      *start -= 1;




536,064✔





4655



    }










4656



  } else {










4657



    if (index <= 0) {




7,307,412✔





4658



      *start = 0;




×







4659



    }










4660












4661



    TSDBKEY* p = taosArrayGet(pDelList, *start);




7,307,412✔





4662



    TSDB_CHECK_NULL(p, code, lino, _end, TSDB_CODE_INVALID_PARA);




7,307,412✔





4663












4664



    while (p->ts <= key && *start < num - 1) {




7,608,948✔





4665



      *start += 1;




301,536✔





4666



    }










4667



  }










4668












4669



_end:




7,307,412✔





4670



  if (code != TSDB_CODE_SUCCESS) {




18,631,282✔





4671



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







4672



  }










4673



  return code;




18,629,573✔





4674



}










4675












4676



int32_t hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t ver, int32_t order,




2,147,483,647✔





4677



                       SVersionRange* pVerRange, bool hasPk, bool* dropped) {










4678



  int32_t code = TSDB_CODE_SUCCESS;




2,147,483,647✔





4679



  int32_t lino = 0;




2,147,483,647✔





4680



  size_t  num = 0;




2,147,483,647✔





4681



  int32_t step = 0;




2,147,483,647✔





4682



  bool    asc = false;




2,147,483,647✔





4683












4684



  *dropped = false;




2,147,483,647✔





4685












4686



  if (pDelList == NULL || (TARRAY_SIZE(pDelList) == 0)) {




2,147,483,647✔





4687



    goto _end;




×







4688



  }










4689












4690



  num = taosArrayGetSize(pDelList);




2,147,483,647✔





4691



  asc = ASCENDING_TRAVERSE(order);




2,147,483,647✔





4692



  step = asc ? 1 : -1;




2,147,483,647✔





4693












4694



  if (hasPk) {  // handle the case where duplicated timestamps existed.




2,147,483,647✔





4695



    code = reverseSearchStartPos(pDelList, *index, key, asc, index);




18,629,573✔





4696



    TSDB_CHECK_CODE(code, lino, _end);




23,376,331✔





4697



  }










4698












4699



  if (asc) {




2,147,483,647✔





4700



    if (*index >= num - 1) {




2,147,483,647✔





4701



      TSDBKEY* last = taosArrayGetLast(pDelList);




2,147,483,647✔





4702



      if (last == NULL) {




2,147,483,647✔





4703



        goto _end;




×







4704



      }










4705












4706



      if (key > last->ts) {




2,147,483,647✔





4707



        goto _end;




2,147,483,647✔





4708



      } else if (key == last->ts) {




10,302,520✔





4709



        TSDBKEY* prev = taosArrayGet(pDelList, num - 2);




10,302,520✔





4710



        if (prev == NULL) {




10,302,520✔





4711



          goto _end;




×







4712



        }










4713












4714



        *dropped = (prev->version >= ver && prev->version <= pVerRange->maxVer && prev->version >= pVerRange->minVer);




10,302,520✔





4715



      }










4716



    } else {










4717



      TSDBKEY* pCurrent = taosArrayGet(pDelList, *index);




2,131,246,421✔





4718



      TSDBKEY* pNext = taosArrayGet(pDelList, (*index) + 1);




2,142,500,280✔





4719



      if (pCurrent == NULL || pNext == NULL) {




2,141,684,345✔





4720



        goto _end;




×







4721



      }










4722












4723



      if (key < pCurrent->ts) {




2,141,684,345✔





4724



        goto _end;




105,866,674✔





4725



      }










4726












4727



      if (pCurrent->ts <= key && pNext->ts >= key && pCurrent->version >= ver &&




2,036,381,587✔





4728



          pVerRange->maxVer >= pCurrent->version) {




1,359,560,683✔





4729



        *dropped = true;




1,359,738,624✔





4730



        goto _end;




1,359,738,318✔





4731



      }










4732












4733



      while (pNext->ts <= key && (*index) < num - 1) {




754,786,698✔





4734



        (*index) += 1;




119,037,642✔





4735












4736



        if ((*index) < num - 1) {




120,412,701✔





4737



          pCurrent = taosArrayGet(pDelList, *index);




100,179,018✔





4738



          pNext = taosArrayGet(pDelList, (*index) + 1);




100,179,292✔





4739



          if (pCurrent == NULL || pNext == NULL) {




100,177,122✔





4740



            break;










4741



          }










4742












4743



          // it is not a consecutive deletion range, ignore it










4744



          if (pCurrent->version == 0 && pNext->version > 0) {




100,178,207✔





4745



            continue;




6,660,629✔





4746



          }










4747












4748



          if (pCurrent->ts <= key && pNext->ts >= key && pCurrent->version >= ver &&




93,517,578✔





4749



              pVerRange->maxVer >= pCurrent->version) {




42,564,642✔





4750



            *dropped = true;




42,564,642✔





4751



            break;




42,563,557✔





4752



          }










4753



        }










4754



      }










4755



    }










4756



  } else {










4757



    if (*index <= 0) {




690,818,290✔





4758



      TSDBKEY* pFirst = taosArrayGet(pDelList, 0);




×







4759



      if (pFirst == NULL) {




×







4760



        goto _end;




×







4761



      }










4762












4763



      if (key < pFirst->ts) {




×







4764



        goto _end;




×







4765



      } else if (key == pFirst->ts) {




×







4766



        *dropped = pFirst->version >= ver;




×







4767



      } else {










4768



        tsdbError("unexpected error, key:%" PRId64 ", first:%" PRId64, key, pFirst->ts);




×







4769



      }










4770



    } else {










4771



      TSDBKEY* pCurrent = taosArrayGet(pDelList, *index);




690,818,290✔





4772



      TSDBKEY* pPrev = taosArrayGet(pDelList, (*index) - 1);




690,818,290✔





4773



      if (pCurrent == NULL || pPrev == NULL) {




690,818,290✔





4774



        goto _end;




×







4775



      }










4776












4777



      if (key > pCurrent->ts) {




690,818,290✔





4778



        goto _end;




503,845,797✔





4779



      }










4780












4781



      if (pPrev->ts <= key && pCurrent->ts >= key && pPrev->version >= ver) {




186,972,493✔





4782



        *dropped = true;




78,089,263✔





4783



        goto _end;




78,089,263✔





4784



      }










4785












4786



      while (pPrev->ts >= key && (*index) > 1) {




162,965,261✔





4787



        (*index) += step;




67,481,912✔





4788












4789



        if ((*index) >= 1) {




67,481,912✔





4790



          pCurrent = taosArrayGet(pDelList, *index);




67,481,912✔





4791



          pPrev = taosArrayGet(pDelList, (*index) - 1);




67,481,912✔





4792



          if (pCurrent == NULL || pPrev == NULL) {




67,481,912✔





4793



            break;










4794



          }










4795












4796



          // it is not a consecutive deletion range, ignore it










4797



          if (pCurrent->version > 0 && pPrev->version == 0) {




67,481,912✔





4798



            continue;




1,981,824✔





4799



          }










4800












4801



          if (pPrev->ts <= key && pCurrent->ts >= key && pPrev->version >= ver) {




65,500,088✔





4802



            *dropped = true;




13,399,881✔





4803



            break;




13,399,881✔





4804



          }










4805



        }










4806



      }










4807



    }










4808



  }










4809












4810



_end:




95,483,349✔





4811



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





4812



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







4813



  }










4814



  return code;




2,147,483,647✔





4815



}










4816












4817



FORCE_INLINE int32_t getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader, TSDBROW** pRes) {










4818



  int32_t  code = TSDB_CODE_SUCCESS;




2,147,483,647✔





4819



  int32_t  lino = 0;




2,147,483,647✔





4820



  int32_t  order = 0;




2,147,483,647✔





4821



  TSDBROW* pRow = NULL;




2,147,483,647✔





4822



  TSDBKEY  key;










4823












4824



  TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





4825



  TSDB_CHECK_NULL(pRes, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





4826












4827



  *pRes = NULL;




2,147,483,647✔





4828












4829



  if (!pIter->hasVal) {




2,147,483,647✔





4830



    goto _end;




2,147,483,647✔





4831



  }










4832












4833



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





4834



  order = pReader->info.order;




2,147,483,647✔





4835



  pRow = tsdbTbDataIterGet(pIter->iter);




2,147,483,647✔





4836












4837



  TSDBROW_INIT_KEY(pRow, key);




2,147,483,647✔





4838



  if (outOfTimeWindow(key.ts, &pReader->info.window)) {




2,147,483,647✔





4839



    pIter->hasVal = false;




266,568✔





4840



    goto _end;




175,776,614✔





4841



  }










4842












4843



  // it is a valid data version










4844



  if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) {




2,147,483,647✔





4845



    if (pDelList == NULL || TARRAY_SIZE(pDelList) == 0) {




2,147,483,647✔





4846



      *pRes = pRow;




2,147,483,647✔





4847



      goto _end;




2,147,483,647✔





4848



    } else {










4849



      bool dropped = false;




2,147,483,647✔





4850



      code = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange,




2,147,483,647✔





4851



                            pReader->suppInfo.numOfPks > 0, &dropped);




2,147,483,647✔





4852



      TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





4853



      if (!dropped) {




2,147,483,647✔





4854



        *pRes = pRow;




2,147,483,647✔





4855



        goto _end;




2,147,483,647✔





4856



      }










4857



    }










4858



  }










4859












4860



  while (1) {










4861



    pIter->hasVal = tsdbTbDataIterNext(pIter->iter);




1,276,632,232✔





4862



    if (!pIter->hasVal) {




1,276,808,317✔





4863



      goto _end;




5,695,762✔





4864



    }










4865












4866



    pRow = tsdbTbDataIterGet(pIter->iter);




1,271,135,547✔





4867












4868



    TSDBROW_INIT_KEY(pRow, key);




1,271,027,859✔





4869



    if (outOfTimeWindow(key.ts, &pReader->info.window)) {




1,270,898,711✔





4870



      pIter->hasVal = false;




38,763✔





4871



      goto _end;




239,657✔





4872



    }










4873












4874



    if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) {




1,270,932,223✔





4875



      if (pDelList == NULL || TARRAY_SIZE(pDelList) == 0) {




548,166,157✔





4876



        *pRes = pRow;




3,146✔





4877



        goto _end;




×







4878



      } else {










4879



        bool dropped = false;




548,165,539✔





4880



        code = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange,




548,221,920✔





4881



                              pReader->suppInfo.numOfPks > 0, &dropped);




548,199,602✔





4882



        TSDB_CHECK_CODE(code, lino, _end);




547,765,110✔





4883



        if (!dropped) {




547,765,110✔





4884



          *pRes = pRow;




14,995,586✔





4885



          goto _end;




14,993,643✔





4886



        }










4887



      }










4888



    }










4889



  }










4890












4891



_end:




2,147,483,647✔





4892



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





4893



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







4894



  }










4895



  return code;




2,147,483,647✔





4896



}










4897












4898



int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey* pCurKey, SArray* pDelList, STsdbReader* pReader) {




2,147,483,647✔





4899



  int32_t     code = TSDB_CODE_SUCCESS;




2,147,483,647✔





4900



  int32_t     lino = 0;




2,147,483,647✔





4901



  SRowMerger* pMerger = NULL;




2,147,483,647✔





4902












4903



  TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





4904



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





4905












4906



  pMerger = &pReader->status.merger;




2,147,483,647✔





4907












4908



  while (1) {




609,620,107✔





4909



    pIter->hasVal = tsdbTbDataIterNext(pIter->iter);




2,147,483,647✔





4910



    if (!pIter->hasVal) {




2,147,483,647✔





4911



      break;




77,679,915✔





4912



    }










4913












4914



    // data exists but not valid










4915



    TSDBROW* pRow = NULL;




2,147,483,647✔





4916



    code = getValidMemRow(pIter, pDelList, pReader, &pRow);




2,147,483,647✔





4917



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





4918



    if (pRow == NULL) {




2,147,483,647✔





4919



      break;




962,737✔





4920



    }










4921












4922



    // ts is not identical, quit










4923



    if (TSDBROW_TS(pRow) != pCurKey->ts) {




2,147,483,647✔





4924



      break;




2,147,483,647✔





4925



    }










4926












4927



    if (pCurKey->numOfPKs > 0) {




597,016,641✔





4928



      SRowKey nextKey = {0};




2,991,120✔





4929



      tRowGetKeyEx(pRow, &nextKey);




2,991,120✔





4930



      if (pkCompEx(pCurKey, &nextKey) != 0) {




2,991,120✔





4931



        break;




1,779,840✔





4932



      }










4933



    }










4934












4935



    STSchema* pTSchema = NULL;




609,557,744✔





4936



    if (pRow->type == TSDBROW_ROW_FMT) {




609,557,744✔





4937



      pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, uid);




599,641,797✔





4938



      TSDB_CHECK_NULL(pTSchema, code, lino, _end, terrno);




599,526,794✔





4939



    }










4940












4941



    code = tsdbRowMergerAdd(pMerger, pRow, pTSchema);




609,371,636✔





4942



    TSDB_CHECK_CODE(code, lino, _end);




609,619,225✔





4943



  }










4944












4945



_end:




2,147,483,647✔





4946



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





4947



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







4948



  }










4949



  return code;




2,147,483,647✔





4950



}










4951












4952



static int32_t doMergeRowsInFileBlockImpl(SBlockData* pBlockData, int32_t* rowIndex, SRowKey* pKey, SRowMerger* pMerger,




2,147,483,647✔





4953



                                          SVersionRange* pVerRange, int32_t step) {










4954



  int32_t code = TSDB_CODE_SUCCESS;




2,147,483,647✔





4955



  int32_t lino = 0;




2,147,483,647✔





4956












4957



  TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





4958



  TSDB_CHECK_NULL(rowIndex, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





4959












4960



  while ((*rowIndex) < pBlockData->nRow && (*rowIndex) >= 0) {




2,147,483,647✔





4961



    SRowKey cur;




2,147,483,647✔





4962



    tColRowGetKey(pBlockData, (*rowIndex), &cur);




2,147,483,647✔





4963



    if (pkCompEx(&cur, pKey) != 0) {




2,147,483,647✔





4964



      break;




2,147,483,647✔





4965



    }










4966












4967



    if (pBlockData->aVersion[(*rowIndex)] > pVerRange->maxVer ||




53,265,913✔





4968



        pBlockData->aVersion[(*rowIndex)] < pVerRange->minVer) {




8,220✔





4969



      (*rowIndex) += step;




×







4970



      continue;




×







4971



    }










4972












4973



    TSDBROW fRow = tsdbRowFromBlockData(pBlockData, (*rowIndex));




8,220✔





4974



    code = tsdbRowMergerAdd(pMerger, &fRow, NULL);




8,220✔





4975



    TSDB_CHECK_CODE(code, lino, _end);




8,220✔





4976



    (*rowIndex) += step;




8,220✔





4977



  }










4978












4979



_end:




×







4980



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





4981



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







4982



  }










4983



  return code;




2,147,483,647✔





4984



}










4985












4986



typedef enum {










4987



  CHECK_FILEBLOCK_CONT = 0x1,










4988



  CHECK_FILEBLOCK_QUIT = 0x2,










4989



} CHECK_FILEBLOCK_STATE;










4990












4991



static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanInfo* pScanInfo,




14,087,353✔





4992



                                         SFileDataBlockInfo* pFBlock, SRowMerger* pMerger, SRowKey* pKey,










4993



                                         CHECK_FILEBLOCK_STATE* state) {










4994



  int32_t             code = TSDB_CODE_SUCCESS;




14,087,353✔





4995



  int32_t             lino = 0;




14,087,353✔





4996



  SFileBlockDumpInfo* pDumpInfo = NULL;




14,087,353✔





4997



  SBlockData*         pBlockData = NULL;




14,087,353✔





4998



  bool                asc = false;




14,087,353✔





4999



  SVersionRange*      pVerRange = NULL;




14,087,353✔





5000



  bool                loadNeighbor = true;




14,087,353✔





5001



  int32_t             step = 0;




14,087,353✔





5002












5003



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




14,087,353✔





5004



  TSDB_CHECK_NULL(state, code, lino, _end, TSDB_CODE_INVALID_PARA);




14,087,353✔





5005












5006



  pDumpInfo = &pReader->status.fBlockDumpInfo;




14,087,353✔





5007



  pBlockData = &pReader->status.fileBlockData;




14,087,353✔





5008



  asc = ASCENDING_TRAVERSE(pReader->info.order);




14,087,353✔





5009



  pVerRange = &pReader->info.verRange;




14,087,353✔





5010



  step = ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1;




14,087,353✔





5011












5012



  *state = CHECK_FILEBLOCK_QUIT;




14,087,353✔





5013



  code = loadNeighborIfOverlap(pFBlock, pScanInfo, pReader, &loadNeighbor);




14,087,353✔





5014



  TSDB_CHECK_CODE(code, lino, _end);




14,087,353✔





5015












5016



  if (loadNeighbor) {




14,087,353✔





5017



    code = doMergeRowsInFileBlockImpl(pBlockData, &pDumpInfo->rowIndex, pKey, pMerger, pVerRange, step);




8,220✔





5018



    TSDB_CHECK_CODE(code, lino, _end);




8,220✔





5019



    if ((pDumpInfo->rowIndex >= pDumpInfo->totalRows && asc) || (pDumpInfo->rowIndex < 0 && !asc)) {




8,220✔





5020



      *state = CHECK_FILEBLOCK_CONT;




×







5021



    }










5022



  }










5023












5024



_end:




14,087,353✔





5025



  if (code != TSDB_CODE_SUCCESS) {




14,087,353✔





5026



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







5027



  }










5028



  return code;




14,087,353✔





5029



}










5030












5031



int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, SRowKey* pKey,




2,147,483,647✔





5032



                                STsdbReader* pReader) {










5033



  int32_t             code = TSDB_CODE_SUCCESS;




2,147,483,647✔





5034



  int32_t             lino = 0;




2,147,483,647✔





5035



  SFileBlockDumpInfo* pDumpInfo = NULL;




2,147,483,647✔





5036



  SRowMerger*         pMerger = NULL;




2,147,483,647✔





5037



  bool                asc = false;




2,147,483,647✔





5038



  int32_t             step = 0;




2,147,483,647✔





5039



  SVersionRange*      pRange = NULL;




2,147,483,647✔





5040












5041



  TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





5042



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





5043












5044



  pDumpInfo = &pReader->status.fBlockDumpInfo;




2,147,483,647✔





5045



  pMerger = &pReader->status.merger;




2,147,483,647✔





5046



  asc = ASCENDING_TRAVERSE(pReader->info.order);




2,147,483,647✔





5047



  step = asc ? 1 : -1;




2,147,483,647✔





5048



  pRange = &pReader->info.verRange;




2,147,483,647✔





5049












5050



  pDumpInfo->rowIndex += step;




2,147,483,647✔





5051



  if ((pDumpInfo->rowIndex <= pBlockData->nRow - 1 && asc) || (pDumpInfo->rowIndex >= 0 && !asc)) {




2,147,483,647✔





5052



    code = doMergeRowsInFileBlockImpl(pBlockData, &pDumpInfo->rowIndex, pKey, pMerger, pRange, step);




2,147,483,647✔





5053



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





5054



  }










5055












5056



  // all rows are consumed, let's try next file block










5057



  if ((pDumpInfo->rowIndex >= pBlockData->nRow && asc) || (pDumpInfo->rowIndex < 0 && !asc)) {




2,147,483,647✔





5058



    while (1) {




×







5059



      SFileDataBlockInfo* pFileBlockInfo = NULL;




14,087,353✔





5060



      code = getCurrentBlockInfo(&pReader->status.blockIter, &pFileBlockInfo, pReader->idStr);




14,087,353✔





5061



      TSDB_CHECK_CODE(code, lino, _end);




14,087,353✔





5062












5063



      if (pFileBlockInfo == NULL) {




14,087,353✔





5064



        break;




×







5065



      }










5066












5067



      CHECK_FILEBLOCK_STATE st = CHECK_FILEBLOCK_QUIT;




14,087,353✔





5068



      code = checkForNeighborFileBlock(pReader, pScanInfo, pFileBlockInfo, pMerger, pKey, &st);




14,087,353✔





5069



      TSDB_CHECK_CODE(code, lino, _end);




14,087,353✔





5070



      if (st == CHECK_FILEBLOCK_QUIT) {




14,087,353✔





5071



        break;




14,087,353✔





5072



      }










5073



    }










5074



  }










5075












5076



_end:




2,147,483,647✔





5077



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





5078



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







5079



  }










5080



  return code;




2,147,483,647✔





5081



}










5082












5083



int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, SRowMerger* pMerger,




2,147,483,647✔





5084



                              int32_t pkSrcSlot, SVersionRange* pVerRange, const char* idStr) {










5085



  int32_t  code = TSDB_CODE_SUCCESS;




2,147,483,647✔





5086



  int32_t  lino = 0;




2,147,483,647✔





5087



  SRowKey* pRowKey = NULL;




2,147,483,647✔





5088



  SRowKey* pNextKey = NULL;




2,147,483,647✔





5089












5090



  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





5091












5092



  pRowKey = &pScanInfo->lastProcKey;




2,147,483,647✔





5093












5094



  while (1) {




2,147,483,647✔





5095



    code = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pkSrcSlot, pVerRange);




2,147,483,647✔





5096



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





5097



    if (!hasDataInSttBlock(pScanInfo)) {




2,147,483,647✔





5098



      break;




82,259,111✔





5099



    }










5100












5101



    pNextKey = getCurrentKeyInSttBlock(pSttBlockReader);




2,147,483,647✔





5102












5103



    int32_t ret = pkCompEx(pRowKey, pNextKey);




2,147,483,647✔





5104



    if (ret == 0) {




2,147,483,647✔





5105



      TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree);




2,147,483,647✔





5106



      code = tsdbRowMergerAdd(pMerger, pRow1, NULL);




2,147,483,647✔





5107



      TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





5108



    } else {










5109



      tsdbTrace("uid:%" PRIu64 " last del index:%d, del range:%d, lastKeyInStt:%" PRId64 ", %s", pScanInfo->uid,




2,147,483,647✔





5110



                pScanInfo->sttBlockDelIndex, (int32_t)taosArrayGetSize(pScanInfo->delSkyline),










5111



                pScanInfo->sttKeyInfo.nextProcKey.ts, idStr);










5112



      break;




2,147,483,647✔





5113



    }










5114



  }










5115












5116



_end:




2,147,483,647✔





5117



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





5118



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







5119



  }










5120



  return code;




2,147,483,647✔





5121



}










5122












5123



int32_t doMergeMemTableMultiRows(TSDBROW* pRow, SRowKey* pKey, uint64_t uid, SIterInfo* pIter, SArray* pDelList,




2,147,483,647✔





5124



                                 TSDBROW* pResRow, STsdbReader* pReader, bool* freeTSRow) {










5125



  int32_t     code = TSDB_CODE_SUCCESS;




2,147,483,647✔





5126



  int32_t     lino = 0;




2,147,483,647✔





5127



  SRowMerger* pMerger = NULL;




2,147,483,647✔





5128



  TSDBROW*    pNextRow = NULL;




2,147,483,647✔





5129



  STSchema*   pTSchema = NULL;




2,147,483,647✔





5130



  STSchema*   pTSchema1 = NULL;




2,147,483,647✔





5131



  TSDBROW     current;




2,147,483,647✔





5132












5133



  TSDB_CHECK_NULL(pRow, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





5134



  TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





5135



  TSDB_CHECK_NULL(pResRow, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





5136



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





5137



  TSDB_CHECK_NULL(freeTSRow, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





5138












5139



  pMerger = &pReader->status.merger;




2,147,483,647✔





5140



  pMerger->arg = pReader->pTsdb->pVnode->pBse;




2,147,483,647✔





5141












5142



  current = *pRow;




2,147,483,647✔





5143












5144



  {  // if the timestamp of the next valid row has a different ts, return current row directly










5145



    pIter->hasVal = tsdbTbDataIterNext(pIter->iter);




2,147,483,647✔





5146












5147



    if (!pIter->hasVal) {




2,147,483,647✔





5148



      *pResRow = *pRow;




2,147,483,647✔





5149



      *freeTSRow = false;




2,147,483,647✔





5150



      goto _end;




2,147,483,647✔





5151



    } else {  // has next point in mem/imem










5152



      code = getValidMemRow(pIter, pDelList, pReader, &pNextRow);




2,147,483,647✔





5153



      TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





5154



      if (pNextRow == NULL) {




2,147,483,647✔





5155



        *pResRow = current;




144,075,240✔





5156



        *freeTSRow = false;




144,081,193✔





5157



        goto _end;




144,038,714✔





5158



      }










5159












5160



      if (TSDBROW_TS(&current) != TSDBROW_TS(pNextRow)) {




2,147,483,647✔





5161



        *pResRow = current;




2,147,483,647✔





5162



        *freeTSRow = false;




2,147,483,647✔





5163



        goto _end;




2,147,483,647✔





5164



      }










5165












5166



      if (pKey->numOfPKs > 0) {




1,443,893,524✔





5167



        SRowKey nextRowKey = {0};




586,704,696✔





5168



        tRowGetKeyEx(pNextRow, &nextRowKey);




586,706,042✔





5169



        if (pkCompEx(pKey, &nextRowKey) != 0) {




586,642,751✔





5170



          *pResRow = current;




583,972,017✔





5171



          *freeTSRow = false;




583,970,727✔





5172



          goto _end;




584,017,860✔





5173



        }










5174



      }










5175



    }










5176



  }










5177












5178



  // start to merge duplicated rows










5179



  if (current.type == TSDBROW_ROW_FMT) {  // get the correct schema for row-wise data in memory




859,712,059✔





5180



    pTSchema = doGetSchemaForTSRow(current.pTSRow->sver, pReader, uid);




843,622,597✔





5181



    TSDB_CHECK_NULL(pTSchema, code, lino, _end, terrno);




844,194,251✔





5182



  }










5183












5184



  code = tsdbRowMergerAdd(pMerger, &current, pTSchema);




860,283,713✔





5185



  TSDB_CHECK_CODE(code, lino, _end);




861,080,505✔





5186












5187



  if (pNextRow->type == TSDBROW_ROW_FMT) {  // get the correct schema for row-wise data in memory




861,080,505✔





5188



    pTSchema1 = doGetSchemaForTSRow(pNextRow->pTSRow->sver, pReader, uid);




845,853,373✔





5189



    TSDB_CHECK_NULL(pTSchema1, code, lino, _end, terrno);




845,981,688✔





5190



  }










5191












5192



  code = tsdbRowMergerAdd(pMerger, pNextRow, pTSchema1);




861,000,590✔





5193



  TSDB_CHECK_CODE(code, lino, _end);




860,936,258✔





5194












5195



  code = doMergeRowsInBuf(pIter, uid, pKey, pDelList, pReader);




860,936,258✔





5196



  TSDB_CHECK_CODE(code, lino, _end);




860,197,316✔





5197












5198



  code = tsdbRowMergerGetRow(pMerger, &pResRow->pTSRow);




860,197,316✔





5199



  TSDB_CHECK_CODE(code, lino, _end);




860,256,863✔





5200












5201



  pResRow->type = TSDBROW_ROW_FMT;




860,256,863✔





5202



  tsdbRowMergerClear(pMerger);




860,640,011✔





5203



  *freeTSRow = true;




860,917,041✔





5204












5205



_end:




2,147,483,647✔





5206



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





5207



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







5208



  }










5209



  return code;




2,147,483,647✔





5210



}










5211












5212



int32_t doMergeMemIMemRows(TSDBROW* pRow, SRowKey* pRowKey, TSDBROW* piRow, SRowKey* piRowKey,




995,810,785✔





5213



                           STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, SRow** pTSRow) {










5214



  int32_t     code = TSDB_CODE_SUCCESS;




995,810,785✔





5215



  int32_t     lino = 0;




995,810,785✔





5216



  SRowMerger* pMerger = NULL;




995,810,785✔





5217



  STSchema*   pSchema = NULL;




995,810,785✔





5218



  STSchema*   piSchema = NULL;




995,810,785✔





5219












5220



  TSDB_CHECK_NULL(pRow, code, lino, _end, TSDB_CODE_INVALID_PARA);




995,810,785✔





5221



  TSDB_CHECK_NULL(piRow, code, lino, _end, TSDB_CODE_INVALID_PARA);




995,810,785✔





5222



  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




995,810,785✔





5223



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




995,810,785✔





5224












5225



  pMerger = &pReader->status.merger;




995,810,785✔





5226



  pMerger->arg = pReader->pTsdb->pVnode->pBse;




996,010,901✔





5227












5228



  if (pRow->type == TSDBROW_ROW_FMT) {




996,077,319✔





5229



    pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid);




996,091,906✔





5230



    TSDB_CHECK_NULL(pSchema, code, lino, _end, terrno);




996,183,847✔





5231



  }










5232












5233



  if (piRow->type == TSDBROW_ROW_FMT) {




996,091,580✔





5234



    piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid);




996,192,772✔





5235



    TSDB_CHECK_NULL(piSchema, code, lino, _end, terrno);




996,182,614✔





5236



  }










5237












5238



  if (ASCENDING_TRAVERSE(pReader->info.order)) {  // ascending order imem --> mem




996,160,346✔





5239



    code = tsdbRowMergerAdd(pMerger, piRow, piSchema);




996,161,521✔





5240



    TSDB_CHECK_CODE(code, lino, _end);




996,202,864✔





5241












5242



    code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, piRowKey, pBlockScanInfo->delSkyline, pReader);




996,202,864✔





5243



    TSDB_CHECK_CODE(code, lino, _end);




996,106,244✔





5244












5245



    code = tsdbRowMergerAdd(pMerger, pRow, pSchema);




996,106,244✔





5246



    TSDB_CHECK_CODE(code, lino, _end);




996,201,129✔





5247












5248



    code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, pRowKey, pBlockScanInfo->delSkyline, pReader);




996,201,129✔





5249



    TSDB_CHECK_CODE(code, lino, _end);




996,196,617✔





5250



  } else {










5251



    code = tsdbRowMergerAdd(pMerger, pRow, pSchema);




19,224✔





5252



    TSDB_CHECK_CODE(code, lino, _end);




19,224✔





5253












5254



    code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, pRowKey, pBlockScanInfo->delSkyline, pReader);




19,224✔





5255



    TSDB_CHECK_CODE(code, lino, _end);




19,224✔





5256












5257



    code = tsdbRowMergerAdd(pMerger, piRow, piSchema);




19,224✔





5258



    TSDB_CHECK_CODE(code, lino, _end);




19,224✔





5259












5260



    code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, piRowKey, pBlockScanInfo->delSkyline, pReader);




19,224✔





5261



    TSDB_CHECK_CODE(code, lino, _end);




19,224✔





5262



  }










5263












5264



  tRowKeyAssign(&pBlockScanInfo->lastProcKey, pRowKey);




996,215,841✔





5265












5266



  code = tsdbRowMergerGetRow(pMerger, pTSRow);




996,212,640✔





5267



  tsdbRowMergerClear(pMerger);




996,216,625✔





5268



  TSDB_CHECK_CODE(code, lino, _end);




996,203,362✔





5269












5270



_end:




996,203,362✔





5271



  if (code != TSDB_CODE_SUCCESS) {




996,203,362✔





5272



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







5273



  }










5274



  return code;




996,185,136✔





5275



}










5276












5277



static int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, TSDBROW* pResRow,




2,147,483,647✔





5278



                                   int64_t endKey, bool* freeTSRow) {










5279



  int32_t    code = TSDB_CODE_SUCCESS;




2,147,483,647✔





5280



  int32_t    lino = 0;




2,147,483,647✔





5281



  TSDBROW*   pRow = NULL;




2,147,483,647✔





5282



  TSDBROW*   piRow = NULL;




2,147,483,647✔





5283



  SArray*    pDelList = NULL;




2,147,483,647✔





5284



  uint64_t   uid = 0;




2,147,483,647✔





5285



  SIterInfo* piter = NULL;




2,147,483,647✔





5286



  SIterInfo* piiter = NULL;




2,147,483,647✔





5287



  SRowKey    rowKey = {0};




2,147,483,647✔





5288



  SRowKey    irowKey = {0};




2,147,483,647✔





5289



  bool       asc = false;




2,147,483,647✔





5290












5291



  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





5292



  TSDB_CHECK_NULL(pResRow, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





5293



  TSDB_CHECK_NULL(freeTSRow, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





5294












5295



  code = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader, &pRow);




2,147,483,647✔





5296



  TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





5297



  code = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader, &piRow);




2,147,483,647✔





5298



  TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





5299












5300



  pDelList = pBlockScanInfo->delSkyline;




2,147,483,647✔





5301



  uid = pBlockScanInfo->uid;




2,147,483,647✔





5302



  piter = &pBlockScanInfo->iter;




2,147,483,647✔





5303



  piiter = &pBlockScanInfo->iiter;




2,147,483,647✔





5304












5305



  // todo refactor










5306



  asc = ASCENDING_TRAVERSE(pReader->info.order);




2,147,483,647✔





5307



  if (piter->hasVal) {




2,147,483,647✔





5308



    tRowGetKeyEx(pRow, &rowKey);




2,147,483,647✔





5309



    if ((rowKey.ts >= endKey && asc) || (rowKey.ts <= endKey && !asc)) {




2,147,483,647✔





5310



      pRow = NULL;




583,859✔





5311



    }










5312



  }










5313












5314



  if (piiter->hasVal) {




2,147,483,647✔





5315



    tRowGetKeyEx(piRow, &irowKey);




2,147,483,647✔





5316



    if ((irowKey.ts >= endKey && asc) || (irowKey.ts <= endKey && !asc)) {




2,147,483,647✔





5317



      piRow = NULL;




×







5318



    }










5319



  }










5320












5321



  if (pRow != NULL && piRow != NULL) {




2,147,483,647✔





5322



    if (rowKey.numOfPKs == 0) {




2,147,483,647✔





5323



      if ((rowKey.ts > irowKey.ts && asc) || (rowKey.ts < irowKey.ts && (!asc))) {  // ik.ts < k.ts




2,147,483,647✔





5324



        code = doMergeMemTableMultiRows(piRow, &irowKey, uid, piiter, pDelList, pResRow, pReader, freeTSRow);




2,147,483,647✔





5325



        TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





5326



      } else if ((rowKey.ts < irowKey.ts && asc) || (rowKey.ts > irowKey.ts && (!asc))) {




996,115,259✔





5327



        code = doMergeMemTableMultiRows(pRow, &rowKey, uid, piter, pDelList, pResRow, pReader, freeTSRow);




38,281✔





5328



        TSDB_CHECK_CODE(code, lino, _end);




38,281✔





5329



      } else {  // ik.ts == k.ts










5330



        pResRow->type = TSDBROW_ROW_FMT;




996,076,978✔





5331



        code = doMergeMemIMemRows(pRow, &rowKey, piRow, &irowKey, pBlockScanInfo, pReader, &pResRow->pTSRow);




995,963,791✔





5332



        TSDB_CHECK_CODE(code, lino, _end);




996,109,951✔





5333



        *freeTSRow = true;




996,109,951✔





5334



      }










5335



    } else {










5336



      int32_t ret = pkCompEx(&rowKey, &irowKey);




×







5337



      if (ret != 0) {




×







5338



        if ((ret > 0 && asc) || (ret < 0 && (!asc))) {  // ik.ts < k.ts




×







5339



          code = doMergeMemTableMultiRows(piRow, &irowKey, uid, piiter, pDelList, pResRow, pReader, freeTSRow);




×







5340



          TSDB_CHECK_CODE(code, lino, _end);




×







5341



        } else if ((ret < 0 && asc) || (ret > 0 && (!asc))) {




×







5342



          code = doMergeMemTableMultiRows(pRow, &rowKey, uid, piter, pDelList, pResRow, pReader, freeTSRow);




×







5343



          TSDB_CHECK_CODE(code, lino, _end);




×







5344



        }










5345



      } else {  // ik.ts == k.ts










5346



        pResRow->type = TSDBROW_ROW_FMT;




×







5347



        code = doMergeMemIMemRows(pRow, &rowKey, piRow, &irowKey, pBlockScanInfo, pReader, &pResRow->pTSRow);




×







5348



        TSDB_CHECK_CODE(code, lino, _end);




×







5349



        *freeTSRow = true;




×







5350



      }










5351



    }










5352



  } else if (piter->hasVal && pRow != NULL) {




2,147,483,647✔





5353



    code = doMergeMemTableMultiRows(pRow, &rowKey, uid, piter, pDelList, pResRow, pReader, freeTSRow);




2,147,483,647✔





5354



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





5355



  } else if (piiter->hasVal && piRow != NULL) {




2,147,483,647✔





5356



    code = doMergeMemTableMultiRows(piRow, &irowKey, uid, piiter, pDelList, pResRow, pReader, freeTSRow);




2,147,483,647✔





5357



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





5358



  }










5359












5360



_end:




2,147,483,647✔





5361



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





5362



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







5363



  }










5364



  return code;




2,147,483,647✔





5365



}










5366












5367



int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, STableBlockScanInfo* pScanInfo) {




2,147,483,647✔





5368



  int32_t             code = TSDB_CODE_SUCCESS;




2,147,483,647✔





5369



  int32_t             lino = 0;




2,147,483,647✔





5370



  int32_t             outputRowIndex = 0;




2,147,483,647✔





5371



  int64_t             uid = 0;




2,147,483,647✔





5372



  SBlockLoadSuppInfo* pSupInfo = NULL;




2,147,483,647✔





5373



  STSchema*           pSchema = NULL;




2,147,483,647✔





5374



  SColVal             colVal = {0};




2,147,483,647✔





5375



  SColVal             dstVal = {0};




2,147,483,647✔





5376



  int32_t             i = 0, j = 0;




2,147,483,647✔





5377












5378



  TSDB_CHECK_NULL(pBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





5379



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





5380



  TSDB_CHECK_NULL(pTSRow, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





5381



  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





5382












5383



  outputRowIndex = pBlock->info.rows;




2,147,483,647✔





5384



  uid = pScanInfo->uid;




2,147,483,647✔





5385












5386



  pSupInfo = &pReader->suppInfo;




2,147,483,647✔





5387



  pSupInfo->args = pReader->pTsdb->pVnode->pBse;




2,147,483,647✔





5388












5389



  pSchema = doGetSchemaForTSRow(pTSRow->sver, pReader, uid);




2,147,483,647✔





5390



  TSDB_CHECK_NULL(pSchema, code, lino, _end, terrno);




2,147,483,647✔





5391












5392



  if (pSupInfo->colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) {




2,147,483,647✔





5393



    SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]);




2,147,483,647✔





5394



    TSDB_CHECK_NULL(pColData, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





5395












5396



    ((int64_t*)pColData->pData)[outputRowIndex] = pTSRow->ts;




2,147,483,647✔





5397



    i += 1;




2,147,483,647✔





5398



  }










5399












5400



  while (i < pSupInfo->numOfCols && j < pSchema->numOfCols) {




2,147,483,647✔





5401



    col_id_t colId = pSupInfo->colId[i];




2,147,483,647✔





5402












5403



    if (colId == pSchema->columns[j].colId) {




2,147,483,647✔





5404



      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]);




2,147,483,647✔





5405



      TSDB_CHECK_NULL(pColInfoData, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





5406












5407



      code = tRowGet(pTSRow, pSchema, j, &colVal);




2,147,483,647✔





5408



      TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





5409



      code = doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo);




2,147,483,647✔





5410



      TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





5411



      i += 1;




2,147,483,647✔





5412



      j += 1;




2,147,483,647✔





5413



    } else if (colId < pSchema->columns[j].colId) {




2,147,483,647✔





5414



      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]);




122,910,000✔





5415



      TSDB_CHECK_NULL(pColInfoData, code, lino, _end, TSDB_CODE_INVALID_PARA);




122,910,000✔





5416












5417



      colDataSetNULL(pColInfoData, outputRowIndex);




122,910,000✔





5418



      i += 1;




122,910,000✔





5419



    } else if (colId > pSchema->columns[j].colId) {




2,147,483,647✔





5420



      j += 1;




2,147,483,647✔





5421



    }










5422



  }










5423












5424



  // set null value since current column does not exist in the "pSchema"










5425



  while (i < pSupInfo->numOfCols) {




2,147,483,647✔





5426



    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]);




124,786,458✔





5427



    TSDB_CHECK_NULL(pColInfoData, code, lino, _end, TSDB_CODE_INVALID_PARA);




124,786,458✔





5428












5429



    colDataSetNULL(pColInfoData, outputRowIndex);




124,786,458✔





5430



    i += 1;




124,786,458✔





5431



  }










5432












5433



  pBlock->info.dataLoad = 1;




2,147,483,647✔





5434



  pBlock->info.rows += 1;




2,147,483,647✔





5435












5436



_end:




2,147,483,647✔





5437



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





5438



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







5439



  }










5440



  return code;




2,147,483,647✔





5441



}










5442



int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData,




2,147,483,647✔





5443



                                 int32_t rowIndex) {










5444



  int32_t             code = TSDB_CODE_SUCCESS;




2,147,483,647✔





5445



  int32_t             lino = 0;




2,147,483,647✔





5446



  int32_t             i = 0, j = 0;




2,147,483,647✔





5447



  int32_t             outputRowIndex = 0;




2,147,483,647✔





5448



  SBlockLoadSuppInfo* pSupInfo = NULL;




2,147,483,647✔





5449



  SColVal             cv = {0};




2,147,483,647✔





5450



  int32_t             numOfInputCols = 0;




2,147,483,647✔





5451



  int32_t             numOfOutputCols = 0;




2,147,483,647✔





5452












5453



  TSDB_CHECK_NULL(pResBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





5454



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





5455



  TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





5456












5457



  outputRowIndex = pResBlock->info.rows;




2,147,483,647✔





5458












5459



  pSupInfo = &pReader->suppInfo;




2,147,483,647✔





5460



  pSupInfo->args = pReader->pTsdb->pVnode->pBse;




2,147,483,647✔





5461












5462



  ((int64_t*)pReader->status.pPrimaryTsCol->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex];




2,147,483,647✔





5463



  i += 1;




2,147,483,647✔





5464












5465



  numOfInputCols = pBlockData->nColData;




2,147,483,647✔





5466



  numOfOutputCols = pSupInfo->numOfCols;




2,147,483,647✔





5467












5468



  while (i < numOfOutputCols && j < numOfInputCols) {




2,147,483,647✔





5469



    SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j);




2,147,483,647✔





5470



    if (pData->cid < pSupInfo->colId[i]) {




2,147,483,647✔





5471



      j += 1;




2,147,483,647✔





5472



      continue;




2,147,483,647✔





5473



    }










5474



    SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->slotId[i]);




2,147,483,647✔





5475



    if (pData->cid == pSupInfo->colId[i]) {




2,147,483,647✔





5476



      code = tColDataGetValue(pData, rowIndex, &cv);




2,147,483,647✔





5477



      TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





5478












5479



      code = doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo);




2,147,483,647✔





5480



      TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





5481



      j += 1;




2,147,483,647✔





5482



    } else if (pData->cid > pCol->info.colId) {




×







5483



      // the specified column does not exist in file block, fill with null data










5484



      colDataSetNULL(pCol, outputRowIndex);




×







5485



    }










5486












5487



    i += 1;




2,147,483,647✔





5488



  }










5489












5490



  while (i < numOfOutputCols) {




2,147,483,647✔





5491



    SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);




20,152,822✔





5492



    TSDB_CHECK_NULL(pCol, code, lino, _end, TSDB_CODE_INVALID_PARA);




20,152,822✔





5493












5494



    colDataSetNULL(pCol, outputRowIndex);




20,152,822✔





5495



    i += 1;




20,152,822✔





5496



  }










5497












5498



  pResBlock->info.dataLoad = 1;




2,147,483,647✔





5499



  pResBlock->info.rows += 1;




2,147,483,647✔





5500












5501



_end:




2,147,483,647✔





5502



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





5503



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







5504



  }










5505



  return code;




2,147,483,647✔





5506



}










5507












5508



int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity,




2,147,483,647✔





5509



                                  STsdbReader* pReader) {










5510



  int32_t      code = TSDB_CODE_SUCCESS;




2,147,483,647✔





5511



  int32_t      lino = 0;




2,147,483,647✔





5512



  SSDataBlock* pBlock = NULL;




2,147,483,647✔





5513












5514



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





5515












5516



  pBlock = pReader->resBlockInfo.pResBlock;




2,147,483,647✔





5517












5518



  do {




2,147,483,647✔





5519



    TSDBROW row = {.type = -1};




2,147,483,647✔





5520



    bool    freeTSRow = false;




2,147,483,647✔





5521



    code = tsdbGetNextRowInMem(pBlockScanInfo, pReader, &row, endKey, &freeTSRow);




2,147,483,647✔





5522



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





5523












5524



    if (row.type == -1) {




2,147,483,647✔





5525



      break;




36,331,131✔





5526



    }










5527












5528



    if (row.type == TSDBROW_ROW_FMT) {




2,147,483,647✔





5529



      code = doAppendRowFromTSRow(pBlock, pReader, row.pTSRow, pBlockScanInfo);




2,147,483,647✔





5530



      if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





5531



        if (freeTSRow) {




×







5532



          taosMemoryFreeClear(row.pTSRow);




×







5533



        }










5534



        TSDB_CHECK_CODE(code, lino, _end);




×







5535



      }










5536



      pBlockScanInfo->lastProcKey.ts = row.pTSRow->ts;




2,147,483,647✔





5537



      pBlockScanInfo->lastProcKey.numOfPKs = row.pTSRow->numOfPKs;




2,147,483,647✔





5538



      if (row.pTSRow->numOfPKs > 0) {




2,147,483,647✔





5539



        code = tRowGetPrimaryKeyDeepCopy(row.pTSRow, &pBlockScanInfo->lastProcKey);




686,942,013✔





5540



        if (code != TSDB_CODE_SUCCESS) {




686,940,740✔





5541



          if (freeTSRow) {




×







5542



            taosMemoryFreeClear(row.pTSRow);




×







5543



          }










5544



          TSDB_CHECK_CODE(code, lino, _end);




×







5545



        }










5546



      }










5547












5548



      if (freeTSRow) {




2,147,483,647✔





5549



        taosMemoryFreeClear(row.pTSRow);




1,856,621,117✔





5550



      }










5551



    } else {










5552



      code = doAppendRowFromFileBlock(pBlock, pReader, row.pBlockData, row.iRow);




427,456,154✔





5553



      TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





5554












5555



      tColRowGetKeyDeepCopy(row.pBlockData, row.iRow, pReader->suppInfo.pkSrcSlot, &pBlockScanInfo->lastProcKey);




2,147,483,647✔





5556



    }










5557












5558



    // no data in buffer, return immediately










5559



    if (!(pBlockScanInfo->iter.hasVal || pBlockScanInfo->iiter.hasVal)) {




2,147,483,647✔





5560



      break;




2,147,483,647✔





5561



    }










5562












5563



    if (pBlock->info.rows >= capacity) {




2,147,483,647✔





5564



      break;




379,600,021✔





5565



    }










5566



  } while (1);










5567












5568



_end:




2,147,483,647✔





5569



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





5570



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







5571



  }










5572



  return code;




2,147,483,647✔





5573



}










5574












5575



// TODO refactor: with createDataBlockScanInfo










5576



int32_t tsdbSetTableList2(void* ptr, const void* pTableList, int32_t num) {




125,655,841✔





5577



  int32_t               code = TSDB_CODE_SUCCESS;




125,655,841✔





5578



  int32_t               lino = 0;




125,655,841✔





5579



  int32_t               size = 0;




125,655,841✔





5580



  STableBlockScanInfo** p = NULL;




125,655,841✔





5581



  STableUidList*        pUidList = NULL;




125,655,841✔





5582



  int32_t               iter = 0;




125,655,841✔





5583



  bool                  acquired = false;




125,667,638✔





5584












5585



  STsdbReader* pReader = (STsdbReader*)ptr;




125,667,638✔





5586



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




125,667,638✔





5587



  size = tSimpleHashGetSize(pReader->status.pTableMap);




125,667,638✔





5588












5589



  code = tsdbAcquireReader(pReader);




125,661,921✔





5590



  TSDB_CHECK_CODE(code, lino, _end);




125,681,855✔





5591



  acquired = true;




125,681,855✔





5592












5593



  while ((p = tSimpleHashIterate(pReader->status.pTableMap, p, &iter)) != NULL) {




251,407,142✔





5594



    clearBlockScanInfo(*p);




125,764,930✔





5595



  }










5596












5597



  tSimpleHashClear(pReader->status.pTableMap);




125,627,078✔





5598












5599



  if (size < num) {




125,639,613✔





5600



    code = ensureBlockScanInfoBuf(&pReader->blockInfoBuf, num);




37,130✔





5601



    TSDB_CHECK_CODE(code, lino, _end);




37,130✔





5602












5603



    char* p1 = taosMemoryRealloc(pReader->status.uidList.tableUidList, sizeof(uint64_t) * num);




37,130✔





5604



    TSDB_CHECK_NULL(p1, code, lino, _end, terrno);




37,130✔





5605












5606



    pReader->status.uidList.tableUidList = (uint64_t*)p1;




37,130✔





5607



  }










5608












5609



  pUidList = &pReader->status.uidList;




125,639,613✔





5610



  pUidList->currentIndex = 0;




125,649,472✔





5611












5612



  STableKeyInfo* pList = (STableKeyInfo*)pTableList;




125,617,915✔





5613



  for (int32_t i = 0; i < num; ++i) {




251,235,970✔





5614



    pUidList->tableUidList[i] = pList[i].uid;




125,666,953✔





5615












5616



    STableBlockScanInfo* pInfo = NULL;




125,625,670✔





5617



    code = getPosInBlockInfoBuf(&pReader->blockInfoBuf, i, &pInfo);




125,639,608✔





5618



    TSDB_CHECK_CODE(code, lino, _end);




125,637,947✔





5619












5620



    code = initTableBlockScanInfo(pInfo, pList[i].uid, pReader->status.pTableMap, pReader);




125,637,947✔





5621



    TSDB_CHECK_CODE(code, lino, _end);




125,577,608✔





5622



  }










5623












5624



_end:




125,581,183✔





5625



  if (code != TSDB_CODE_SUCCESS) {




125,582,880✔





5626



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







5627



  }










5628



  if (acquired) {




125,595,213✔





5629



    (void)tsdbReleaseReader(pReader);




125,595,213✔





5630



  }










5631



  return code;




125,693,853✔





5632



}










5633












5634



uint64_t tsdbGetReaderMaxVersion2(STsdbReader* pReader) { return pReader->info.verRange.maxVer; }




×







5635












5636



static int32_t doOpenReaderImpl(STsdbReader* pReader) {




1,716,009,737✔





5637



  int32_t         code = TSDB_CODE_SUCCESS;




1,716,009,737✔





5638



  int32_t         lino = 0;




1,716,009,737✔





5639



  SReaderStatus*  pStatus = NULL;




1,716,009,737✔





5640



  SDataBlockIter* pBlockIter = NULL;




1,716,009,737✔





5641












5642



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,716,009,737✔





5643












5644



  pStatus = &pReader->status;




1,716,009,737✔





5645



  pBlockIter = &pStatus->blockIter;




1,716,118,589✔





5646












5647



  if (pReader->bFilesetDelimited) {




1,716,207,705✔





5648



    getMemTableTimeRange(pReader, &pReader->status.memTableMaxKey, &pReader->status.memTableMinKey);




293,844,596✔





5649



    pReader->status.bProcMemFirstFileset = true;




293,972,812✔





5650



  }










5651












5652



  code = initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader);




1,716,567,751✔





5653



  TSDB_CHECK_CODE(code, lino, _end);




1,714,405,359✔





5654












5655



  code = resetDataBlockIterator(&pStatus->blockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo),




1,714,610,816✔





5656



                                pReader->idStr);




1,714,405,359✔





5657



  TSDB_CHECK_CODE(code, lino, _end);




1,715,985,032✔





5658












5659



  if (pStatus->fileIter.numOfFiles == 0) {




1,715,985,032✔





5660



    pStatus->loadFromFile = false;




1,324,699,343✔





5661



  } else {










5662



    code = initForFirstBlockInFile(pReader, pBlockIter);




390,999,659✔





5663



  }










5664












5665



  if (!pStatus->loadFromFile) {




1,715,230,395✔





5666



    code = resetTableListIndex(pStatus, pReader->idStr);




1,326,203,801✔





5667



    TSDB_CHECK_CODE(code, lino, _end);




1,325,996,285✔





5668



  }










5669












5670



_end:




1,715,139,429✔





5671



  if (code != TSDB_CODE_SUCCESS) {




1,715,139,429✔





5672



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







5673



  }










5674



  return code;




1,714,022,043✔





5675



}










5676












5677



static void freeSchemaFunc(void* param) {




85,665,802✔





5678



  void** p = (void**)param;




85,665,802✔





5679



  taosMemoryFreeClear(*p);




85,665,802✔





5680



}




85,648,999✔





5681












5682



static void clearSharedPtr(STsdbReader* p) {




45,672,694✔





5683



  if (p) {




45,672,694✔





5684



    p->status.pTableMap = NULL;




45,672,694✔





5685



    p->status.uidList.tableUidList = NULL;




45,672,694✔





5686



    p->info.pSchema = NULL;




45,672,694✔





5687



    p->pReadSnap = NULL;




45,672,694✔





5688



    p->pSchemaMap = NULL;




45,672,694✔





5689



  }










5690



}




45,672,694✔





5691












5692



static int32_t setSharedPtr(STsdbReader* pDst, const STsdbReader* pSrc) {




45,672,694✔





5693



  pDst->status.pTableMap = pSrc->status.pTableMap;




45,672,694✔





5694



  pDst->status.uidList = pSrc->status.uidList;




45,672,694✔





5695



  pDst->info.pSchema = pSrc->info.pSchema;




45,672,694✔





5696



  pDst->pSchemaMap = pSrc->pSchemaMap;




45,672,694✔





5697



  pDst->pReadSnap = pSrc->pReadSnap;




45,672,694✔





5698



  pDst->pReadSnap->pfSetArray = pSrc->pReadSnap->pfSetArray;




45,672,694✔





5699












5700



  if (pDst->info.pSchema) {




45,672,694✔





5701



    return tsdbRowMergerInit(&pDst->status.merger, pDst->info.pSchema);




45,672,694✔





5702



  }










5703












5704



  return TSDB_CODE_SUCCESS;




×







5705



}










5706












5707



// ====================================== EXPOSED APIs ======================================










5708



int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables,




1,845,265,981✔





5709



                        SSDataBlock* pResBlock, void** ppReader, const char* idstr, SHashObj** pIgnoreTables) {










5710



  int32_t      code = TSDB_CODE_SUCCESS;




1,845,265,981✔





5711



  int32_t      lino = 0;




1,845,265,981✔





5712



  STimeWindow  window = {0};




1,845,265,981✔





5713



  SVnodeCfg*   pConf = NULL;




1,845,265,981✔





5714



  STsdbReader* pReader = NULL;




1,845,265,981✔





5715



  int32_t      capacity = 0;




1,845,265,981✔





5716












5717



  window = pCond->twindows;




1,845,265,981✔





5718



  pConf = &(((SVnode*)pVnode)->config);




1,845,823,470✔





5719












5720



  capacity = pConf->tsdbCfg.maxRows;




1,845,236,516✔





5721



  if (pResBlock != NULL) {




1,845,623,146✔





5722



    code = blockDataEnsureCapacity(pResBlock, capacity);




1,837,863,679✔





5723



    TSDB_CHECK_CODE(code, lino, _end);




1,838,226,362✔





5724



  }










5725












5726



  code = tsdbReaderCreate(pVnode, pCond, ppReader, capacity, pResBlock, idstr);




1,845,985,829✔





5727



  TSDB_CHECK_CODE(code, lino, _end);




1,844,084,023✔





5728












5729



  // check for query time window










5730



  pReader = *ppReader;




1,844,084,023✔





5731



  if (isEmptyQueryTimeWindow(&pReader->info.window) && pCond->type == TIMEWINDOW_RANGE_CONTAINED) {




1,844,470,684✔





5732



    tsdbDebug("%p query window not overlaps with the data set, no result returned, %s", pReader, pReader->idStr);




14,063,253✔





5733



    goto _end;




14,072,311✔





5734



  }










5735












5736



  if (pCond->type == TIMEWINDOW_RANGE_EXTERNAL) {




1,829,486,508✔





5737



    // update the SQueryTableDataCond to create inner reader










5738



    int32_t order = pCond->order;




22,836,347✔





5739



    if (order == TSDB_ORDER_ASC) {




22,836,347✔





5740



      pCond->twindows = pCond->extTwindows[0];




22,836,347✔





5741



      pCond->order = TSDB_ORDER_DESC;




22,836,347✔





5742



    } else {










5743



      pCond->twindows = pCond->extTwindows[1];




×







5744



      pCond->order = TSDB_ORDER_ASC;




×







5745



    }










5746












5747



    // here we only need one more row, so the capacity is set to be ONE.










5748



    code = tsdbReaderCreate(pVnode, pCond, (void**)&((STsdbReader*)pReader)->innerReader[0], 1, pResBlock, idstr);




22,836,347✔





5749



    TSDB_CHECK_CODE(code, lino, _end);




22,836,347✔





5750












5751



    if (order == TSDB_ORDER_ASC) {




22,836,347✔





5752



      pCond->twindows = pCond->extTwindows[1];




22,836,347✔





5753



    } else {










5754



      pCond->twindows = pCond->extTwindows[0];




×







5755



    }










5756



    pCond->order = order;




22,836,347✔





5757












5758



    code = tsdbReaderCreate(pVnode, pCond, (void**)&((STsdbReader*)pReader)->innerReader[1], 1, pResBlock, idstr);




22,836,347✔





5759



    pCond->twindows = window;




22,836,347✔





5760



    TSDB_CHECK_CODE(code, lino, _end);




22,836,347✔





5761



  }










5762












5763



  // NOTE: the endVersion in pCond is the data version not schema version, so pCond->endVersion is not correct here.










5764



  //  no valid error code set in metaGetTbTSchema, so let's set the error code here.










5765



  //  we should proceed in case of tmq processing.










5766



  if (pCond->suid != 0) {




1,831,241,567✔





5767



    code = metaGetTbTSchemaMaybeNull(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, -1, 1, &pReader->info.pSchema);




1,464,126,981✔





5768



    TSDB_CHECK_CODE(code, lino, _end);




1,463,446,146✔





5769



    if (pReader->info.pSchema == NULL) {




1,463,446,146✔





5770



      tsdbWarn("failed to get table schema, suid:%" PRIu64 ", ver:-1, %s", pReader->info.suid, pReader->idStr);




441,556✔





5771



    }










5772



  } else if (numOfTables > 0) {




366,775,292✔





5773



    STableKeyInfo* pKey = pTableList;




366,995,729✔





5774



    code = metaGetTbTSchemaMaybeNull(pReader->pTsdb->pVnode->pMeta, pKey->uid, -1, 1, &pReader->info.pSchema);




366,995,729✔





5775



    TSDB_CHECK_CODE(code, lino, _end);




366,924,467✔





5776



    if (pReader->info.pSchema == NULL) {




366,924,467✔





5777



      tsdbWarn("failed to get table schema, uid:%" PRIu64 ", ver:-1, %s", pKey->uid, pReader->idStr);




×







5778



    }










5779



  }










5780












5781



  if (pReader->info.pSchema != NULL) {




1,830,738,273✔





5782



    code = tsdbRowMergerInit(&pReader->status.merger, pReader->info.pSchema);




1,830,931,067✔





5783



    TSDB_CHECK_CODE(code, lino, _end);




1,830,336,130✔





5784



  }










5785












5786



  pReader->pSchemaMap = tSimpleHashInit(8, taosFastHash);




1,830,665,354✔





5787



  if (pReader->pSchemaMap == NULL) {




1,830,363,979✔





5788



    tsdbError("failed init schema hash for reader %s", pReader->idStr);




×







5789



    TSDB_CHECK_NULL(pReader->pSchemaMap, code, lino, _end, terrno);




×







5790



  }










5791












5792



  tSimpleHashSetFreeFp(pReader->pSchemaMap, freeSchemaFunc);




1,830,691,716✔





5793



  if (pReader->info.pSchema != NULL) {




1,829,937,179✔





5794



    code = updateBlockSMAInfo(pReader->info.pSchema, &pReader->suppInfo);




1,830,539,621✔





5795



    TSDB_CHECK_CODE(code, lino, _end);




1,830,333,593✔





5796



  }










5797












5798



  STsdbReader* p = (pReader->innerReader[0] != NULL) ? pReader->innerReader[0] : pReader;




1,829,854,518✔





5799












5800



  code = createDataBlockScanInfo(p, &pReader->blockInfoBuf, pTableList, &pReader->status.uidList, numOfTables,




1,830,221,389✔





5801



                                 &pReader->status.pTableMap);










5802



  TSDB_CHECK_CODE(code, lino, _end);




1,830,458,214✔





5803












5804



  pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES);




1,830,458,214✔





5805



  TSDB_CHECK_NULL(pReader->status.pLDataIterArray, code, lino, _end, terrno);




1,831,138,053✔





5806












5807



  pReader->flag = READER_STATUS_SUSPEND;




1,830,896,071✔





5808



  pReader->info.execMode = pCond->notLoadData ? READER_EXEC_ROWS : READER_EXEC_DATA;




1,830,861,188✔





5809












5810



  pReader->pIgnoreTables = pIgnoreTables;




1,831,591,999✔





5811



  tsdbDebug("%p total numOfTable:%d, window:%" PRId64 " - %" PRId64 ", verRange:%" PRId64 " - %" PRId64




1,830,153,518✔





5812



            " in this query %s",










5813



            pReader, numOfTables, pReader->info.window.skey, pReader->info.window.ekey, pReader->info.verRange.minVer,










5814



            pReader->info.verRange.maxVer, pReader->idStr);










5815












5816



  // NOTE: simulate the suspend operation after the reader is suspended, ref: TD-38007.










5817



  // tsdbReaderSuspend2(pReader);










5818












5819



_end:




1,744,483,865✔





5820



  if (code != TSDB_CODE_SUCCESS) {




1,845,048,659✔





5821



    tsdbError("%s failed at line %d since %s, %s", __func__, lino, tstrerror(code), idstr);




49,150✔





5822



    tsdbReaderClose2(*ppReader);




49,150✔





5823



    *ppReader = NULL;  // reset the pointer value.




49,150✔





5824



  }










5825



  return code;




1,845,048,659✔





5826



}










5827












5828



void tsdbReaderClose2(void* ptr) {




2,147,483,647✔





5829



  if (ptr == NULL) {




2,147,483,647✔





5830



    return;




658,029,259✔





5831



  }










5832



  STsdbReader* pReader = (STsdbReader*)ptr;




1,891,665,180✔





5833



  int32_t code = tsdbAcquireReader(pReader);




1,891,665,180✔





5834



  if (code) {




1,891,777,293✔





5835



    return;




×







5836



  }










5837












5838



  {










5839



    if (pReader->innerReader[0] != NULL || pReader->innerReader[1] != NULL) {




1,891,777,293✔





5840



      STsdbReader* p = pReader->innerReader[0];




22,844,624✔





5841



      clearSharedPtr(p);




22,836,347✔





5842












5843



      p = pReader->innerReader[1];




22,836,347✔





5844



      clearSharedPtr(p);




22,836,347✔





5845












5846



      tsdbReaderClose2(pReader->innerReader[0]);




22,836,347✔





5847



      tsdbReaderClose2(pReader->innerReader[1]);




22,836,347✔





5848



    }










5849



  }










5850












5851



  SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;




1,891,787,180✔





5852



  TARRAY2_DESTROY(&pSupInfo->colAggArray, NULL);




1,891,769,909✔





5853












5854



  if (pSupInfo->buildBuf) {




1,891,746,038✔





5855



    for (int32_t i = 0; i < pSupInfo->numOfCols; ++i) {




2,147,483,647✔





5856



      if (pSupInfo->buildBuf[i] != NULL) {




2,147,483,647✔





5857



        taosMemoryFreeClear(pSupInfo->buildBuf[i]);




881,515,259✔





5858



      }










5859



    }










5860



  }










5861












5862



  if (pReader->resBlockInfo.freeBlock) {




1,891,484,540✔





5863



    blockDataDestroy(pReader->resBlockInfo.pResBlock);




7,638,191✔





5864



    pReader->resBlockInfo.pResBlock = NULL;




7,638,191✔





5865



  }










5866












5867



  taosMemoryFree(pSupInfo->colId);




1,891,718,699✔





5868



  tBlockDataDestroy(&pReader->status.fileBlockData);




1,891,663,475✔





5869



  cleanupDataBlockIterator(&pReader->status.blockIter, shouldFreePkBuf(&pReader->suppInfo));




1,891,395,204✔





5870












5871



  size_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap);




1,891,618,919✔





5872



  if (pReader->status.pTableMap != NULL) {




1,891,716,750✔





5873



    destroyAllBlockScanInfo(&pReader->status.pTableMap);




1,831,927,418✔





5874



  }










5875



  clearBlockScanInfoBuf(&pReader->blockInfoBuf);




1,891,509,749✔





5876












5877



  if (pReader->pFileReader != NULL) {




1,891,387,452✔





5878



    tsdbDataFileReaderClose(&pReader->pFileReader);




34,998,844✔





5879



  }










5880












5881



  SReadCostSummary* pCost = &pReader->cost;




1,891,402,548✔





5882



  SFilesetIter*     pFilesetIter = &pReader->status.fileIter;




1,891,624,634✔





5883



  if (pFilesetIter->pSttBlockReader != NULL) {




1,891,562,442✔





5884



    SSttBlockReader* pSttBlockReader = pFilesetIter->pSttBlockReader;




1,716,526,035✔





5885



    tMergeTreeClose(&pSttBlockReader->mergeTree);




1,716,622,204✔





5886












5887



    clearRowKey(&pSttBlockReader->currentKey);




1,716,432,717✔





5888



    taosMemoryFree(pSttBlockReader);




1,716,430,945✔





5889



  }










5890












5891



  destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost);




1,891,396,089✔





5892



  pReader->status.pLDataIterArray = NULL;




1,891,186,059✔





5893



  taosMemoryFreeClear(pReader->status.uidList.tableUidList);




1,891,244,419✔





5894












5895



  tsdbTrace("tsdb/reader-close: %p, untake snapshot", pReader);




1,891,457,657✔





5896



  void* p = pReader->pReadSnap;




1,891,460,369✔





5897



  if ((p == atomic_val_compare_exchange_ptr((void**)&pReader->pReadSnap, p, NULL)) && (p != NULL)) {




1,891,299,146✔





5898



    tsdbUntakeReadSnap2(pReader, p, true);




1,681,290,464✔





5899



    pReader->pReadSnap = NULL;




1,680,662,623✔





5900



  }










5901












5902



  code = tsem_destroy(&pReader->resumeAfterSuspend);




1,891,237,315✔





5903



  if (code != 0) {




1,891,347,617✔





5904



    tsdbWarn("faild to destroy tsem"); 




×







5905



  }










5906



  (void)tsdbReleaseReader(pReader);




1,891,347,617✔





5907



  (void)tsdbUninitReaderLock(pReader);




1,891,397,022✔





5908












5909



  tsdbDebug(




1,891,548,302✔





5910



      "%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64










5911



      " SMA-time:%.2f ms, fileBlocks:%" PRId64










5912



      ", fileBlocks-load-time:%.2f ms, "










5913



      "build in-memory-block-time:%.2f ms, sttBlocks:%" PRId64 ", sttBlocks-time:%.2f ms, sttStatisBlock:%" PRId64










5914



      ", stt-statis-Block-time:%.2f ms, composed-blocks:%" PRId64










5915



      ", composed-blocks-time:%.2fms, STableBlockScanInfo size:%.2f Kb, createTime:%.2f ms,createSkylineIterTime:%.2f "










5916



      "ms, initSttBlockReader:%.2fms, %s",










5917



      pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaDataLoad, pCost->smaLoadTime, pCost->numOfBlocks,










5918



      pCost->blockLoadTime, pCost->buildmemBlock, pCost->sttCost.loadBlocks, pCost->sttCost.blockElapsedTime,










5919



      pCost->sttCost.loadStatisBlocks, pCost->sttCost.statisElapsedTime, pCost->composedBlocks,










5920



      pCost->buildComposedBlockTime, numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pCost->createScanInfoList,










5921



      pCost->createSkylineIterTime, pCost->initSttBlockReader, pReader->idStr);










5922












5923



  taosMemoryFree(pReader->idStr);




1,891,469,142✔





5924












5925



  tsdbRowMergerCleanup(&pReader->status.merger);




1,891,250,023✔





5926



  taosMemoryFree(pReader->info.pSchema);




1,891,157,807✔





5927












5928



  tSimpleHashCleanup(pReader->pSchemaMap);




1,891,165,652✔





5929



  taosMemoryFreeClear(pReader);




1,891,164,115✔





5930



}










5931












5932



static int32_t doSuspendCurrentReader(STsdbReader* pCurrentReader) {




×







5933



  int32_t               code = TSDB_CODE_SUCCESS;




×







5934



  int32_t               lino = 0;




×







5935



  SReaderStatus*        pStatus = NULL;




×







5936



  STableBlockScanInfo** p = NULL;




×







5937












5938



  TSDB_CHECK_NULL(pCurrentReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




×







5939












5940



  pStatus = &pCurrentReader->status;




×







5941












5942



  if (pStatus->loadFromFile) {




×







5943



    tsdbDataFileReaderClose(&pCurrentReader->pFileReader);




×







5944












5945



    SReadCostSummary* pCost = &pCurrentReader->cost;




×







5946



    destroySttBlockReader(pStatus->pLDataIterArray, &pCost->sttCost);




×







5947



    pStatus->pLDataIterArray = taosArrayInit(4, POINTER_BYTES);




×







5948



    TSDB_CHECK_NULL(pStatus->pLDataIterArray, code, lino, _end, terrno);




×







5949



  }










5950












5951



  // resetDataBlockScanInfo excluding lastKey










5952



  int32_t step = ASCENDING_TRAVERSE(pCurrentReader->info.order) ? 1 : -1;




×







5953



  int32_t iter = 0;




×







5954



  while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) {




×







5955



    STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p;




×







5956



    clearBlockScanInfoLoadInfo(pInfo);




×







5957



  }










5958












5959



  pStatus->uidList.currentIndex = 0;




×







5960



  initReaderStatus(pStatus);




×







5961












5962



_end:




×







5963



  if (code != TSDB_CODE_SUCCESS) {




×







5964



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







5965



  }










5966



  return code;




×







5967



}










5968












5969



int32_t tsdbReaderSuspend2(STsdbReader* pReader) {




×







5970



  int32_t code = TSDB_CODE_SUCCESS;




×







5971



  int32_t lino = 0;




×







5972












5973



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




×







5974












5975



  // save reader's base state & reset top state to be reconstructed from base state










5976



  pReader->status.suspendInvoked = true;  // record the suspend status




×







5977












5978



  if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) {




×







5979



    if (pReader->step == EXTERNAL_ROWS_PREV) {




×







5980



      code = doSuspendCurrentReader(pReader->innerReader[0]);




×







5981



    } else if (pReader->step == EXTERNAL_ROWS_MAIN) {




×







5982



      code = doSuspendCurrentReader(pReader);




×







5983



    } else {










5984



      code = doSuspendCurrentReader(pReader->innerReader[1]);




×







5985



    }










5986



  } else {










5987



    code = doSuspendCurrentReader(pReader);




×







5988



  }










5989












5990



  // make sure only release once










5991



  void* p = pReader->pReadSnap;




×







5992



  if ((p == atomic_val_compare_exchange_ptr((void**)&pReader->pReadSnap, p, NULL)) && (p != NULL)) {




×







5993



    tsdbUntakeReadSnap2(pReader, p, false);




×







5994



  }










5995












5996



  if (pReader->bFilesetDelimited) {




×







5997



    pReader->status.memTableMinKey = INT64_MAX;




×







5998



    pReader->status.memTableMaxKey = INT64_MIN;




×







5999



  }










6000



  pReader->flag = READER_STATUS_SUSPEND;




×







6001












6002



  if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) {




×







6003



    clearSharedPtr(pReader->innerReader[0]);




×







6004



    clearSharedPtr(pReader->innerReader[1]);




×







6005



  }










6006












6007



#if SUSPEND_RESUME_TEST










6008



  tsem_post(&pReader->resumeAfterSuspend);










6009



#endif










6010












6011



  tsdbDebug("reader: %p suspended in this query %s, step:%d", pReader, pReader->idStr, pReader->step);




×







6012












6013



_end:




×







6014



  if (code != TSDB_CODE_SUCCESS) {




×







6015



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







6016



  }










6017



  return code;




×







6018



}










6019












6020



static int32_t tsdbSetQueryReseek(void* pQHandle) {




×







6021



  int32_t      code = 0;




×







6022



  STsdbReader* pReader = pQHandle;




×







6023












6024



  code = tsdbTryAcquireReader(pReader);




×







6025



  if (code == 0) {




×







6026



    if (pReader->flag == READER_STATUS_SUSPEND) {




×







6027



      code = tsdbReleaseReader(pReader);




×







6028



      return code;




×







6029



    }










6030












6031



    code = tsdbReaderSuspend2(pReader);




×







6032



    (void)tsdbReleaseReader(pReader);




×







6033



    return code;




×







6034



  } else if (code == EBUSY) {




×







6035



    return TSDB_CODE_VND_QUERY_BUSY;




×







6036



  } else {










6037



    terrno = TAOS_SYSTEM_ERROR(code);




×







6038



    return TSDB_CODE_FAILED;




×







6039



  }










6040



}










6041












6042



int32_t tsdbReaderResume2(STsdbReader* pReader) {




1,831,311,938✔





6043



  int32_t               code = TSDB_CODE_SUCCESS;




1,831,311,938✔





6044



  int32_t               lino = 0;




1,831,311,938✔





6045



  STableBlockScanInfo** pBlockScanInfo = NULL;




1,831,311,938✔





6046



  int32_t               numOfTables = 0;




1,831,311,938✔





6047












6048



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,831,311,938✔





6049












6050



  pBlockScanInfo = pReader->status.pTableIter;




1,831,311,938✔





6051












6052



  //  restore reader's state, task snapshot










6053



  numOfTables = tSimpleHashGetSize(pReader->status.pTableMap);




1,831,898,728✔





6054



  if (numOfTables > 0) {




1,831,131,010✔





6055



    tsdbTrace("tsdb/reader: %p, take snapshot", pReader);




1,679,971,526✔





6056



    code = tsdbTakeReadSnap2(pReader, tsdbSetQueryReseek, &pReader->pReadSnap, pReader->idStr);




1,679,971,526✔





6057



    TSDB_CHECK_CODE(code, lino, _end);




1,679,975,063✔





6058












6059



    // open reader failure may cause the flag still to be READER_STATUS_SUSPEND, which may cause suspend reader failure.










6060



    // So we need to set it A.S.A.P










6061



    pReader->flag = READER_STATUS_NORMAL;




1,679,975,063✔





6062












6063



    if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) {




1,680,105,864✔





6064



      code = doOpenReaderImpl(pReader);




1,657,105,884✔





6065



      TSDB_CHECK_CODE(code, lino, _end);




1,657,300,897✔





6066



    } else {










6067



      STsdbReader* pPrevReader = pReader->innerReader[0];




22,836,347✔





6068



      STsdbReader* pNextReader = pReader->innerReader[1];




22,836,347✔





6069












6070



      // we need only one row










6071



      pPrevReader->resBlockInfo.capacity = 1;




22,836,347✔





6072



      code = setSharedPtr(pPrevReader, pReader);




22,836,347✔





6073



      TSDB_CHECK_CODE(code, lino, _end);




22,836,248✔





6074












6075



      pNextReader->resBlockInfo.capacity = 1;




22,836,248✔





6076



      code = setSharedPtr(pNextReader, pReader);




22,836,347✔





6077



      TSDB_CHECK_CODE(code, lino, _end);




22,836,347✔





6078












6079



      if (pReader->step == 0 || pReader->step == EXTERNAL_ROWS_PREV) {




22,836,347✔





6080



        code = doOpenReaderImpl(pPrevReader);




22,836,347✔





6081



        TSDB_CHECK_CODE(code, lino, _end);




22,836,347✔





6082



      } else if (pReader->step == EXTERNAL_ROWS_MAIN) {




×







6083



        code = doOpenReaderImpl(pReader);




×







6084



        TSDB_CHECK_CODE(code, lino, _end);




×







6085



      } else {










6086



        code = doOpenReaderImpl(pNextReader);




×







6087



        TSDB_CHECK_CODE(code, lino, _end);




×







6088



      }










6089



    }










6090



  }










6091












6092



  tsdbDebug("reader: %p resumed uid %" PRIu64 ", numOfTable:%" PRId32 ", in this query %s", pReader,




1,831,296,728✔





6093



            pBlockScanInfo ? (*pBlockScanInfo)->uid : 0, numOfTables, pReader->idStr);










6094












6095



_end:




1,745,632,865✔





6096



  if (code != TSDB_CODE_SUCCESS) {




1,831,302,518✔





6097



    tsdbError("%s failed at line %d since %s, %s", __func__, lino, tstrerror(code),




×







6098



              (pReader && pReader->idStr) ? pReader->idStr : "");










6099



  }










6100



  return code;




1,831,073,105✔





6101



}










6102












6103



static int32_t buildFromPreFilesetBuffer(STsdbReader* pReader) {




4,406,937✔





6104



  int32_t        code = TSDB_CODE_SUCCESS;




4,406,937✔





6105



  int32_t        lino = 0;




4,406,937✔





6106



  SReaderStatus* pStatus = NULL;




4,406,937✔





6107



  SSDataBlock*   pBlock = NULL;




4,406,937✔





6108



  int32_t        fid = 0;




4,406,937✔





6109



  STimeWindow    win = {0};




4,406,937✔





6110












6111



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




4,406,937✔





6112












6113



  pStatus = &pReader->status;




4,406,937✔





6114



  pBlock = pReader->resBlockInfo.pResBlock;




4,406,937✔





6115



  fid = pReader->status.pCurrentFileset->fid;




4,406,937✔





6116



  tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &win.skey, &win.ekey);




4,406,937✔





6117












6118



  int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? win.skey : win.ekey;




4,406,937✔





6119



  code = buildBlockFromBufferSeqForPreFileset(pReader, endKey);




4,406,937✔





6120



  TSDB_CHECK_CODE(code, lino, _end);




4,406,937✔





6121



  if (pBlock->info.rows <= 0) {




4,406,937✔





6122



    tsdbDebug("finished pre-fileset %d buffer processing. %s", fid, pReader->idStr);




2,281,604✔





6123



    pStatus->bProcMemPreFileset = false;




2,281,604✔





6124



    pStatus->processingMemPreFileSet = false;




2,281,604✔





6125



    if (pReader->notifyFn) {




2,281,604✔





6126



      STsdReaderNotifyInfo info = {0};




2,261,996✔





6127



      info.duration.filesetId = fid;




2,261,996✔





6128



      pReader->notifyFn(TSD_READER_NOTIFY_DURATION_START, &info, pReader->notifyParam);




2,261,996✔





6129



      tsdbDebug("new duration %d start notification when buffer pre-fileset, %s", fid, pReader->idStr);




2,261,996✔





6130



    }










6131



  }










6132












6133



_end:




4,406,937✔





6134



  if (code != TSDB_CODE_SUCCESS) {




4,406,937✔





6135



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







6136



  }










6137



  return code;




4,406,937✔





6138



}










6139












6140



static int32_t doTsdbNextDataBlockFilesetDelimited(STsdbReader* pReader) {




1,623,778,111✔





6141



  int32_t        code = TSDB_CODE_SUCCESS;




1,623,778,111✔





6142



  int32_t        lino = 0;




1,623,778,111✔





6143



  SReaderStatus* pStatus = NULL;




1,623,778,111✔





6144



  SSDataBlock*   pBlock = NULL;




1,623,778,111✔





6145












6146



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,623,778,111✔





6147












6148



  pStatus = &pReader->status;




1,623,778,111✔





6149



  pBlock = pReader->resBlockInfo.pResBlock;




1,623,913,894✔





6150












6151



  if (pStatus->loadFromFile) {




1,623,906,796✔





6152



    if (pStatus->bProcMemPreFileset) {




239,516,432✔





6153



      code = buildFromPreFilesetBuffer(pReader);




2,161,421✔





6154



      TSDB_CHECK_CODE(code, lino, _end);




2,161,421✔





6155



      if (pBlock->info.rows > 0) {




2,161,421✔





6156



        goto _end;




642,624✔





6157



      }










6158



    }










6159












6160



    code = buildBlockFromFiles(pReader);




238,861,564✔





6161



    TSDB_CHECK_CODE(code, lino, _end);




238,737,022✔





6162












6163



    tsdbTrace("block from file rows: %" PRId64 ", will process pre-file set buffer: %d. %s", pBlock->info.rows,




238,737,022✔





6164



              pStatus->bProcMemFirstFileset, pReader->idStr);










6165



    if (pStatus->bProcMemPreFileset) {




238,737,938✔





6166



      if (pBlock->info.rows > 0) {




1,482,709✔





6167



        if (pReader->notifyFn && !pReader->status.processingMemPreFileSet) {




1,482,709✔





6168



          int32_t              fid = pReader->status.pCurrentFileset->fid;




×







6169



          STsdReaderNotifyInfo info = {0};




×







6170



          info.duration.filesetId = fid;




×







6171



          pReader->notifyFn(TSD_READER_NOTIFY_NEXT_DURATION_BLOCK, &info, pReader->notifyParam);




×







6172



        }










6173



      } else {










6174



        pStatus->bProcMemPreFileset = false;




×







6175



      }










6176



    }










6177












6178



    if (pBlock->info.rows <= 0) {




238,764,723✔





6179



      code = resetTableListIndex(&pReader->status, pReader->idStr);




40,094,277✔





6180



      TSDB_CHECK_CODE(code, lino, _end);




40,097,817✔





6181












6182



      int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN;




40,097,817✔





6183



      code = buildBlockFromBufferSequentially(pReader, endKey);




40,095,252✔





6184



      TSDB_CHECK_CODE(code, lino, _end);




40,086,293✔





6185



    }










6186



  } else {  // no data in files, let's try the buffer










6187



    int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN;




1,384,314,106✔





6188



    code = buildBlockFromBufferSequentially(pReader, endKey);




1,384,415,129✔





6189



    TSDB_CHECK_CODE(code, lino, _end);




1,384,270,509✔





6190



  }










6191












6192



_end:




1,384,270,509✔





6193



  if (code != TSDB_CODE_SUCCESS) {




1,623,682,430✔





6194



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







6195



  }










6196



  return code;




1,623,689,305✔





6197



}










6198












6199



static int32_t doTsdbNextDataBlockFilesFirst(STsdbReader* pReader) {




2,147,483,647✔





6200



  int32_t        code = TSDB_CODE_SUCCESS;




2,147,483,647✔





6201



  int32_t        lino = 0;




2,147,483,647✔





6202



  SReaderStatus* pStatus = NULL;




2,147,483,647✔





6203



  SSDataBlock*   pBlock = NULL;




2,147,483,647✔





6204












6205



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





6206












6207



  pStatus = &pReader->status;




2,147,483,647✔





6208



  pBlock = pReader->resBlockInfo.pResBlock;




2,147,483,647✔





6209












6210



  if (pStatus->loadFromFile) {




2,147,483,647✔





6211



    code = buildBlockFromFiles(pReader);




2,147,483,647✔





6212



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





6213












6214



    if (pBlock->info.rows <= 0) {




2,147,483,647✔





6215



      code = resetTableListIndex(&pReader->status, pReader->idStr);




378,228,554✔





6216



      TSDB_CHECK_CODE(code, lino, _end);




378,202,553✔





6217












6218



      int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN;




378,202,553✔





6219



      code = buildBlockFromBufferSequentially(pReader, endKey);




378,204,561✔





6220



      TSDB_CHECK_CODE(code, lino, _end);




378,181,157✔





6221



    }










6222



  } else {  // no data in files, let's try the buffer










6223



    int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN;




2,147,483,647✔





6224



    code = buildBlockFromBufferSequentially(pReader, endKey);




2,147,483,647✔





6225



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





6226



  }










6227












6228



_end:




2,147,483,647✔





6229



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





6230



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







6231



  }










6232



  return code;




2,147,483,647✔





6233



}










6234












6235



static int32_t doTsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) {




2,147,483,647✔





6236



  int32_t      code = TSDB_CODE_SUCCESS;




2,147,483,647✔





6237



  int32_t      lino = 0;




2,147,483,647✔





6238



  SSDataBlock* pBlock = NULL;




2,147,483,647✔





6239












6240



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





6241












6242



  // cleanup the data that belongs to the previous data block










6243



  pBlock = pReader->resBlockInfo.pResBlock;




2,147,483,647✔





6244



  blockDataCleanup(pBlock);




2,147,483,647✔





6245












6246



  *hasNext = false;




2,147,483,647✔





6247












6248



  SReaderStatus* pStatus = &pReader->status;




2,147,483,647✔





6249



  if (tSimpleHashGetSize(pStatus->pTableMap) == 0) {




2,147,483,647✔





6250



    goto _end;




150,825,649✔





6251



  }










6252












6253



  if (!pReader->bFilesetDelimited) {




2,147,483,647✔





6254



    code = doTsdbNextDataBlockFilesFirst(pReader);




2,147,483,647✔





6255



  } else {










6256



    code = doTsdbNextDataBlockFilesetDelimited(pReader);




1,624,127,318✔





6257



  }










6258












6259



  *hasNext = pBlock->info.rows > 0;




2,147,483,647✔





6260



  TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





6261












6262



_end:




2,147,483,647✔





6263



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





6264



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







6265



  }










6266



  return code;




2,147,483,647✔





6267



}










6268












6269



int32_t tsdbNextDataBlock2(void* p, bool* hasNext) {




2,147,483,647✔





6270



  int32_t        code = TSDB_CODE_SUCCESS;




2,147,483,647✔





6271



  int32_t        lino = 0;




2,147,483,647✔





6272



  SReaderStatus* pStatus = NULL;




2,147,483,647✔





6273



  bool           acquired = false;




2,147,483,647✔





6274












6275



  STsdbReader* pReader = (STsdbReader*)p;




2,147,483,647✔





6276



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





6277



  TSDB_CHECK_NULL(hasNext, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





6278












6279



  *hasNext = false;




2,147,483,647✔





6280












6281



  code = pReader->code;




2,147,483,647✔





6282



  TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





6283












6284



  if ((pReader->type != TIMEWINDOW_RANGE_EXTERNAL && isEmptyQueryTimeWindow(&pReader->info.window)) || pReader->step == EXTERNAL_ROWS_NEXT) {




2,147,483,647✔





6285



    goto _end;




31,062,164✔





6286



  }










6287












6288



  pStatus = &pReader->status;




2,147,483,647✔





6289












6290



  // NOTE: the following codes is used to perform test for suspend/resume for tsdbReader when it blocks the commit










6291



  // the data should be ingested in round-robin and all the child tables should be createted before ingesting data










6292



  // the version range of query will be used to identify the correctness of suspend/resume functions.










6293



  // this function will be blocked before loading the SECOND block from vnode-buffer, and restart itself from sst-files










6294



#if SUSPEND_RESUME_TEST










6295



  if (!pReader->status.suspendInvoked && !pReader->status.loadFromFile) {










6296



    tsem_wait(&pReader->resumeAfterSuspend);










6297



  }










6298



#endif










6299












6300



  code = tsdbAcquireReader(pReader);




2,147,483,647✔





6301



  TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





6302



  acquired = true;




2,147,483,647✔





6303












6304



  tsdbTrace("tsdb/read: %p, take read mutex, code: %d", pReader, code);




2,147,483,647✔





6305












6306



  if (pReader->flag == READER_STATUS_SUSPEND) {




2,147,483,647✔





6307



    code = tsdbReaderResume2(pReader);




1,763,573,373✔





6308



    TSDB_CHECK_CODE(code, lino, _end);




1,763,364,886✔





6309



  }










6310












6311



  if (pReader->innerReader[0] != NULL && pReader->step == 0) {




2,147,483,647✔





6312



    if (isEmptyQueryTimeWindow(&pReader->innerReader[0]->info.window)) {




22,836,347✔





6313



      *hasNext = false;




5,082,772✔





6314



    } else {










6315



      code = doTsdbNextDataBlock2(pReader->innerReader[0], hasNext);




17,753,575✔





6316



      TSDB_CHECK_CODE(code, lino, _end);




17,753,575✔





6317



    }










6318



    










6319



    pReader->step = EXTERNAL_ROWS_PREV;




22,836,347✔





6320



    if (*hasNext) {




22,836,347✔





6321



      pStatus = &pReader->innerReader[0]->status;




16,133,001✔





6322



      if (pStatus->composedDataBlock) {




16,133,001✔





6323



        tsdbTrace("tsdb/read: %p, unlock read mutex", pReader);




16,133,001✔





6324



        code = tsdbReleaseReader(pReader);




16,133,001✔





6325



        acquired = false;




16,133,001✔





6326



        TSDB_CHECK_CODE(code, lino, _end);




16,133,001✔





6327



      }










6328












6329



      return code;




16,133,001✔





6330



    }










6331



  }










6332












6333



  if (pReader->step == EXTERNAL_ROWS_PREV) {




2,147,483,647✔





6334



    if (!isEmptyQueryTimeWindow(&pReader->info.window)) {




22,836,347✔





6335



      // prepare for the main scan










6336



      if (tSimpleHashGetSize(pReader->status.pTableMap) > 0) {




16,550,237✔





6337



        code = doOpenReaderImpl(pReader);




16,550,237✔





6338



      }










6339












6340



      int32_t step = 1;




16,550,237✔





6341



      resetAllDataBlockScanInfo(pReader->status.pTableMap, pReader->innerReader[0]->info.window.ekey, step);




16,550,237✔





6342



      TSDB_CHECK_CODE(code, lino, _end);




16,550,237✔





6343



    }










6344



    










6345



    pReader->step = EXTERNAL_ROWS_MAIN;




22,836,347✔





6346



  }










6347












6348












6349



  if (!isEmptyQueryTimeWindow(&pReader->info.window)) {




2,147,483,647✔





6350



    code = doTsdbNextDataBlock2(pReader, hasNext);




2,147,483,647✔





6351



    TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





6352



  } else {










6353



    *hasNext = false;




6,286,110✔





6354



  }










6355



  










6356



  if (*hasNext) {




2,147,483,647✔





6357



    if (pStatus->composedDataBlock) {




2,147,483,647✔





6358



      tsdbTrace("tsdb/read: %p, unlock read mutex", pReader);




2,147,483,647✔





6359



      code = tsdbReleaseReader(pReader);




2,147,483,647✔





6360



      acquired = false;




2,147,483,647✔





6361



      TSDB_CHECK_CODE(code, lino, _end);




2,147,483,647✔





6362



    }










6363



    return code;




2,147,483,647✔





6364



  }










6365












6366



  if (pReader->step == EXTERNAL_ROWS_MAIN && pReader->innerReader[1] != NULL) {




1,819,504,784✔





6367



    if (isEmptyQueryTimeWindow(&pReader->innerReader[1]->info.window)) {




22,836,347✔





6368



      *hasNext = false;




4,031,279✔





6369



    } else {










6370



      // prepare for the next row scan










6371



      if (tSimpleHashGetSize(pReader->status.pTableMap) > 0) {




18,805,068✔





6372



        code = doOpenReaderImpl(pReader->innerReader[1]);




18,805,068✔





6373



      }










6374












6375



      int32_t step = -1;




18,805,068✔





6376



      resetAllDataBlockScanInfo(pReader->innerReader[1]->status.pTableMap, pReader->info.window.ekey, step);




18,805,068✔





6377



      TSDB_CHECK_CODE(code, lino, _end);




18,805,068✔





6378












6379



      code = doTsdbNextDataBlock2(pReader->innerReader[1], hasNext);




18,805,068✔





6380



      TSDB_CHECK_CODE(code, lino, _end);




18,805,068✔





6381



    }










6382



    










6383



    pReader->step = EXTERNAL_ROWS_NEXT;




22,836,347✔





6384



    if (*hasNext) {




22,836,347✔





6385



      pStatus = &pReader->innerReader[1]->status;




17,003,432✔





6386



      if (pStatus->composedDataBlock) {




17,003,432✔





6387



        tsdbTrace("tsdb/read: %p, unlock read mutex", pReader);




17,003,432✔





6388



        code = tsdbReleaseReader(pReader);




17,003,432✔





6389



        acquired = false;




17,003,432✔





6390



        TSDB_CHECK_CODE(code, lino, _end);




17,003,432✔





6391



      }










6392












6393



      return code;




17,003,432✔





6394



    }










6395



  }










6396












6397



  tsdbTrace("tsdb/read: %p, unlock read mutex", pReader);




1,801,621,211✔





6398



  code = tsdbReleaseReader(pReader);




1,801,621,617✔





6399



  acquired = false;




1,801,474,592✔





6400



  TSDB_CHECK_CODE(code, lino, _end);




1,801,474,592✔





6401












6402



_end:




1,801,474,592✔





6403



  if (code != TSDB_CODE_SUCCESS) {




1,832,536,756✔





6404



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







6405



  }










6406



  if (acquired) {




1,832,547,015✔





6407



    tsdbTrace("tsdb/read: %p, unlock read mutex", pReader);




×







6408



    (void)tsdbReleaseReader(pReader);




×







6409



  }










6410



  return code;




1,832,553,554✔





6411



}










6412












6413



static int32_t doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols,




189,249,370✔





6414



                                SColumnDataAgg* pTsAgg) {










6415



  // do fill all null column value SMA info










6416



  int32_t code = TSDB_CODE_SUCCESS;




189,249,370✔





6417



  int32_t lino = 0;




189,249,370✔





6418



  int32_t i = 0, j = 0;




189,249,370✔





6419



  int32_t size = 0;




189,249,370✔





6420












6421



  TSDB_CHECK_NULL(pSup, code, lino, _end, TSDB_CODE_INVALID_PARA);




189,249,370✔





6422












6423



  size = (int32_t)TARRAY2_SIZE(&pSup->colAggArray);




189,249,370✔





6424



  code = TARRAY2_INSERT_PTR(&pSup->colAggArray, 0, pTsAgg);




189,259,737✔





6425



  TSDB_CHECK_CODE(code, lino, _end);




189,268,530✔





6426












6427



  size++;




189,268,530✔





6428












6429



  while (j < numOfCols && i < size) {




1,154,536,246✔





6430



    SColumnDataAgg* pAgg = &pSup->colAggArray.data[i];




965,277,296✔





6431



    if (pAgg->colId == pSup->colId[j]) {




965,279,212✔





6432



      i += 1;




622,745,535✔





6433



      j += 1;




622,745,535✔





6434



    } else if (pAgg->colId < pSup->colId[j]) {




342,555,882✔





6435



      i += 1;




342,552,050✔





6436



    } else if (pSup->colId[j] < pAgg->colId) {




×







6437



      if (pSup->colId[j] != PRIMARYKEY_TIMESTAMP_COL_ID) {




×







6438



        SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows};




×







6439



        code = TARRAY2_INSERT_PTR(&pSup->colAggArray, i, &nullColAgg);




×







6440



        TSDB_CHECK_CODE(code, lino, _end);




×







6441












6442



        i += 1;




×







6443



        size++;




×







6444



      }










6445



      j += 1;




×







6446



    }










6447



  }










6448












6449



  while (j < numOfCols) {




189,263,330✔





6450



    if (pSup->colId[j] != PRIMARYKEY_TIMESTAMP_COL_ID) {




4,380✔





6451



      SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows};




4,380✔





6452



      code = TARRAY2_INSERT_PTR(&pSup->colAggArray, i, &nullColAgg);




4,380✔





6453



      TSDB_CHECK_CODE(code, lino, _end);




4,380✔





6454












6455



      i += 1;




4,380✔





6456



    }










6457



    j++;




4,380✔





6458



  }










6459












6460



_end:




189,258,950✔





6461



  if (code != TSDB_CODE_SUCCESS) {




189,258,950✔





6462



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







6463



  }










6464



  return code;




189,245,538✔





6465



}










6466












6467



int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave, bool* hasNullSMA) {




1,407,942,943✔





6468



  int32_t             code = TSDB_CODE_SUCCESS;




1,407,942,943✔





6469



  int32_t             lino = 0;




1,407,942,943✔





6470



  SColumnDataAgg**    pBlockSMA = NULL;




1,407,942,943✔





6471



  SFileDataBlockInfo* pBlockInfo = NULL;




1,407,942,943✔





6472












6473



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,408,015,263✔





6474



  TSDB_CHECK_NULL(pDataBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,408,015,263✔





6475



  TSDB_CHECK_NULL(allHave, code, lino, _end, TSDB_CODE_INVALID_PARA);




1,408,015,263✔





6476












6477



  pBlockSMA = &pDataBlock->pBlockAgg;




1,408,015,263✔





6478



  *allHave = false;




1,408,023,928✔





6479



  *pBlockSMA = NULL;




1,408,182,027✔





6480












6481



  if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) {




1,408,057,283✔





6482



    goto _end;




25,005✔





6483



  }










6484












6485



  // there is no statistics data for composed block










6486



  if (pReader->status.composedDataBlock || (!pReader->suppInfo.smaValid)) {




1,407,877,746✔





6487



    goto _end;




1,218,629,663✔





6488



  }










6489












6490



  code = getCurrentBlockInfo(&pReader->status.blockIter, &pBlockInfo, pReader->idStr);




189,255,426✔





6491



  TSDB_CHECK_CODE(code, lino, _end);




189,257,195✔





6492












6493



  SBlockLoadSuppInfo* pSup = &pReader->suppInfo;




189,257,195✔





6494












6495



  SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock;




189,257,195✔





6496



  if (pResBlock->info.id.uid != pBlockInfo->uid) {




189,257,195✔





6497



    goto _end;




×







6498



  }










6499












6500



  //  int64_t st = taosGetTimestampUs();










6501



  TARRAY2_CLEAR(&pSup->colAggArray, 0);




189,255,118✔





6502












6503



  SBrinRecord pRecord;




189,251,594✔





6504



  blockInfoToRecord(&pRecord, pBlockInfo, pSup);




189,262,782✔





6505



  code = tsdbDataFileReadBlockSma(pReader->pFileReader, &pRecord, &pSup->colAggArray);




189,266,614✔





6506



  if (code != TSDB_CODE_SUCCESS) {




189,250,157✔





6507



    tsdbDebug("vgId:%d, failed to load block SMA for uid %" PRIu64 ", code:%s, %s", 0, pBlockInfo->uid, tstrerror(code),




×







6508



              pReader->idStr);










6509



    TSDB_CHECK_CODE(code, lino, _end);




×







6510



  }










6511












6512



  if (pSup->colAggArray.size > 0) {




189,250,157✔





6513



    *allHave = true;




189,253,989✔





6514



  } else {










6515



    *pBlockSMA = NULL;




×







6516



    goto _end;




×







6517



  }










6518












6519



  // always load the first primary timestamp column data










6520



  SColumnDataAgg* pTsAgg = &pSup->tsColAgg;




189,255,905✔





6521












6522



  pTsAgg->numOfNull = 0;




189,247,762✔





6523



  pTsAgg->colId = PRIMARYKEY_TIMESTAMP_COL_ID;




189,255,118✔





6524



  pTsAgg->min = pResBlock->info.window.skey;




189,255,118✔





6525



  pTsAgg->max = pResBlock->info.window.ekey;




189,251,594✔





6526












6527



  // update the number of NULL data rows










6528



  size_t numOfCols = pSup->numOfCols;




189,257,034✔





6529












6530



  if (pResBlock->pBlockAgg == NULL) {




189,253,510✔





6531



    size_t num = taosArrayGetSize(pResBlock->pDataBlock);




189,259,737✔





6532



    pResBlock->pBlockAgg = taosMemoryCalloc(num, sizeof(SColumnDataAgg));




189,253,202✔





6533



    TSDB_CHECK_NULL(pResBlock->pBlockAgg, code, lino, _end, terrno);




189,244,409✔





6534



    for (int i = 0; i < num; ++i) {




851,663,847✔





6535



      pResBlock->pBlockAgg[i].colId = -1;




662,412,903✔





6536



    }










6537



  }










6538












6539



  // do fill all null column value SMA info










6540



  code = doFillNullColSMA(pSup, pBlockInfo->numRow, numOfCols, pTsAgg);




189,252,073✔





6541



  TSDB_CHECK_CODE(code, lino, _end);




189,249,370✔





6542












6543



  size_t size = pSup->colAggArray.size;




189,249,370✔





6544












6545



  int32_t i = 0, j = 0;




189,251,286✔





6546



  while (j < numOfCols && i < size) {




1,154,539,121✔





6547



    SColumnDataAgg* pAgg = &pSup->colAggArray.data[i];




965,221,415✔





6548



    if (pAgg->colId == pSup->colId[j]) {




965,225,247✔





6549



      pResBlock->pBlockAgg[pSup->slotId[j]] = *pAgg;




622,711,927✔





6550



      i += 1;




622,751,831✔





6551



      j += 1;




622,751,831✔





6552



    } else if (pAgg->colId < pSup->colId[j]) {




342,535,696✔





6553



      i += 1;




342,536,004✔





6554



    } else if (pSup->colId[j] < pAgg->colId) {




×







6555



      pResBlock->pBlockAgg[pSup->slotId[j]].colId = -1;




×







6556



      *allHave = false;




×







6557



      j += 1;




×







6558



    }










6559



  }










6560












6561



  *pBlockSMA = pResBlock->pBlockAgg;




189,317,706✔





6562



  pReader->cost.smaDataLoad += 1;




189,255,118✔





6563












6564



  //  double elapsedTime = (taosGetTimestampUs() - st) / 1000.0;










6565



  pReader->cost.smaLoadTime += 0;  // elapsedTime;




189,260,866✔





6566












6567



  tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", %s", 0, pBlockInfo->uid, pReader->idStr);




189,264,698✔





6568












6569



_end:




1,407,895,550✔





6570



  if (code != TSDB_CODE_SUCCESS) {




1,408,031,728✔





6571



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







6572



  }










6573



  return code;




1,408,031,728✔





6574



}










6575












6576



static int32_t doRetrieveDataBlock(STsdbReader* pReader, SSDataBlock** pBlock) {




423,417,322✔





6577



  int32_t              code = TSDB_CODE_SUCCESS;




423,417,322✔





6578



  int32_t              lino = 0;




423,417,322✔





6579



  SReaderStatus*       pStatus = NULL;




423,417,322✔





6580



  SFileDataBlockInfo*  pBlockInfo = NULL;




423,417,322✔





6581



  STableBlockScanInfo* pBlockScanInfo = NULL;




423,489,763✔





6582



  bool                 reset = false;




423,466,407✔





6583












6584



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




423,466,407✔





6585



  TSDB_CHECK_NULL(pBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);




423,466,407✔





6586












6587



  pStatus = &pReader->status;




423,466,407✔





6588



  *pBlock = NULL;




423,550,436✔





6589












6590



  code = getCurrentBlockInfo(&pStatus->blockIter, &pBlockInfo, pReader->idStr);




423,547,215✔





6591



  TSDB_CHECK_CODE(code, lino, _end);




423,513,202✔





6592












6593



  code = pReader->code;




423,513,202✔





6594



  TSDB_CHECK_CODE(code, lino, _end);




423,544,566✔





6595












6596



  code = getTableBlockScanInfo(pStatus->pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr);




423,544,566✔





6597



  TSDB_CHECK_CODE(code, lino, _end);




423,596,391✔





6598












6599



  reset = true;




423,596,391✔





6600



  code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData, pBlockScanInfo->uid);




423,596,391✔





6601



  TSDB_CHECK_CODE(code, lino, _end);




423,477,767✔





6602












6603



  code = copyBlockDataToSDataBlock(pReader, &pBlockScanInfo->lastProcKey);




423,477,767✔





6604



  TSDB_CHECK_CODE(code, lino, _end);




423,514,183✔





6605












6606



  *pBlock = pReader->resBlockInfo.pResBlock;




423,514,183✔





6607












6608



_end:




423,497,008✔





6609



  if (code != TSDB_CODE_SUCCESS) {




423,549,769✔





6610



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







6611



    if (reset) {




×







6612



      tBlockDataReset(&pStatus->fileBlockData);




×







6613



    }










6614



  }










6615



  return code;




423,549,769✔





6616



}










6617












6618



int32_t tsdbRetrieveDataBlock2(void* p, SSDataBlock** pBlock, SArray* pIdList) {




2,147,483,647✔





6619



  int32_t      code = TSDB_CODE_SUCCESS;




2,147,483,647✔





6620



  int32_t      lino = 0;




2,147,483,647✔





6621



  STsdbReader* pTReader = NULL;




2,147,483,647✔





6622












6623



  STsdbReader* pReader = (STsdbReader*)p;




2,147,483,647✔





6624



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





6625



  TSDB_CHECK_NULL(pBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,147,483,647✔





6626












6627



  *pBlock = NULL;




2,147,483,647✔





6628












6629



  pTReader = pReader;




2,147,483,647✔





6630



  if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) {




2,147,483,647✔





6631



    if (pReader->step == EXTERNAL_ROWS_PREV) {




44,271,778✔





6632



      pTReader = pReader->innerReader[0];




16,133,001✔





6633



    } else if (pReader->step == EXTERNAL_ROWS_NEXT) {




28,138,777✔





6634



      pTReader = pReader->innerReader[1];




17,003,432✔





6635



    }










6636



  }










6637












6638



  SReaderStatus* pStatus = &pTReader->status;




2,147,483,647✔





6639



  if (pStatus->composedDataBlock || pReader->info.execMode == READER_EXEC_ROWS) {




2,147,483,647✔





6640



    //    tsdbReaderSuspend2(pReader);










6641



    //    tsdbReaderResume2(pReader);










6642



    *pBlock = pTReader->resBlockInfo.pResBlock;




2,147,483,647✔





6643



    goto _end;




2,147,483,647✔





6644



  }










6645












6646



  code = doRetrieveDataBlock(pTReader, pBlock);




423,417,110✔





6647












6648



  tsdbTrace("tsdb/read-retrieve: %p, unlock read mutex", pReader);




423,439,313✔





6649



  (void)tsdbReleaseReader(pReader);




423,439,313✔





6650



  TSDB_CHECK_CODE(code, lino, _end);




423,422,878✔





6651












6652



  //  tsdbReaderSuspend2(pReader);










6653



  //  tsdbReaderResume2(pReader);










6654



_end:




423,422,878✔





6655



  if (code != TSDB_CODE_SUCCESS) {




2,147,483,647✔





6656



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







6657



  }










6658



  return code;




2,147,483,647✔





6659



}










6660












6661



int32_t tsdbReaderReset2(void* p, SQueryTableDataCond* pCond) {




127,530,221✔





6662



  int32_t code = TSDB_CODE_SUCCESS;




127,530,221✔





6663



  int32_t lino = 0;




127,530,221✔





6664



  bool    acquired = false;




127,530,221✔





6665












6666



  STsdbReader* pReader = (STsdbReader*)p;




127,530,221✔





6667



  tsdbTrace("tsdb/reader-reset: %p, take read mutex", pReader);




127,530,221✔





6668



  code = tsdbAcquireReader(pReader);




127,530,221✔





6669



  TSDB_CHECK_CODE(code, lino, _end);




127,533,188✔





6670



  acquired = true;




127,533,188✔





6671












6672



  pReader->step = 0;




127,533,188✔





6673



  if (pReader->flag == READER_STATUS_SUSPEND) {




127,533,188✔





6674



    code = tsdbReaderResume2(pReader);




67,321,357✔





6675



    TSDB_CHECK_CODE(code, lino, _end);




67,321,357✔





6676



  }










6677












6678



  if (isEmptyQueryTimeWindow(&pReader->info.window) || pReader->pReadSnap == NULL) {




127,533,188✔





6679



    tsdbDebug("tsdb reader reset return %p, %s", pReader->pReadSnap, pReader->idStr);




222,154✔





6680



    code = tsdbReleaseReader(pReader);




222,154✔





6681



    acquired = false;




222,154✔





6682



    TSDB_CHECK_CODE(code, lino, _end);




222,154✔





6683



    goto _end;




222,154✔





6684



  }










6685












6686



  SReaderStatus*  pStatus = &pReader->status;




127,311,034✔





6687



  SDataBlockIter* pBlockIter = &pStatus->blockIter;




127,311,034✔





6688












6689



  TSDB_CHECK_NULL(pCond, code, lino, _end, TSDB_CODE_INVALID_PARA);




127,311,034✔





6690



  pReader->info.order = pCond->order;




127,311,034✔





6691



  pReader->type = TIMEWINDOW_RANGE_CONTAINED;




127,308,977✔





6692



  code = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows, &pReader->info.window);




127,311,034✔





6693



  TSDB_CHECK_CODE(code, lino, _end);




127,309,926✔





6694



  pStatus->loadFromFile = true;




127,309,926✔





6695



  pStatus->pTableIter = NULL;




127,311,034✔





6696












6697



  // allocate buffer in order to load data blocks from file










6698



  memset(&pReader->suppInfo.tsColAgg, 0, sizeof(SColumnDataAgg));




127,309,926✔





6699












6700



  pReader->suppInfo.tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID;




127,311,034✔





6701



  tsdbDataFileReaderClose(&pReader->pFileReader);




127,311,034✔





6702












6703



  int32_t numOfTables = tSimpleHashGetSize(pStatus->pTableMap);




127,308,977✔





6704












6705



  code = initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader);




127,311,034✔





6706



  TSDB_CHECK_CODE(code, lino, _end);




127,307,290✔





6707












6708



  code = resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo), pReader->idStr);




127,307,290✔





6709



  TSDB_CHECK_CODE(code, lino, _end);




127,302,344✔





6710












6711



  code = resetTableListIndex(&pReader->status, pReader->idStr);




127,302,344✔





6712



  TSDB_CHECK_CODE(code, lino, _end);




127,302,370✔





6713












6714



  bool    asc = ASCENDING_TRAVERSE(pReader->info.order);




127,302,370✔





6715



  int32_t step = asc ? 1 : -1;




127,298,361✔





6716



  int64_t ts = 0;




127,298,361✔





6717



  if (asc) {




127,298,361✔





6718



    ts = (pReader->info.window.skey > INT64_MIN) ? pReader->info.window.skey - 1 : pReader->info.window.skey;




124,013,000✔





6719



  } else {










6720



    ts = (pReader->info.window.ekey < INT64_MAX) ? pReader->info.window.ekey + 1 : pReader->info.window.ekey;




3,285,361✔





6721



  }










6722



  resetAllDataBlockScanInfo(pStatus->pTableMap, ts, step);




127,297,277✔





6723












6724



  // no data in files, let's try buffer in memory










6725



  if (pStatus->fileIter.numOfFiles == 0) {




127,288,319✔





6726



    pStatus->loadFromFile = false;




76,781,499✔





6727



    code = resetTableListIndex(pStatus, pReader->idStr);




76,786,618✔





6728



    TSDB_CHECK_CODE(code, lino, _end);




76,789,332✔





6729



  } else {










6730



    code = initForFirstBlockInFile(pReader, pBlockIter);




50,495,615✔





6731



    if (code != TSDB_CODE_SUCCESS) {




50,451,642✔





6732



      tsdbError("%p reset reader failed, numOfTables:%d, query range:%" PRId64 " - %" PRId64 " in query %s", pReader,




×







6733



                numOfTables, pReader->info.window.skey, pReader->info.window.ekey, pReader->idStr);










6734



      TSDB_CHECK_CODE(code, lino, _end);




×







6735



    }










6736



  }










6737












6738



  tsdbDebug("%p reset reader, suid:%" PRIu64 ", numOfTables:%d, skey:%" PRId64 ", query range:%" PRId64 " - %" PRId64




127,240,974✔





6739



            " in query %s",










6740



            pReader, pReader->info.suid, numOfTables, pCond->twindows.skey, pReader->info.window.skey,










6741



            pReader->info.window.ekey, pReader->idStr);










6742












6743



  code = tsdbReleaseReader(pReader);




127,240,974✔





6744



  acquired = false;




127,251,887✔





6745



  TSDB_CHECK_CODE(code, lino, _end);




127,251,887✔





6746












6747



_end:




127,251,887✔





6748



  if (code != TSDB_CODE_SUCCESS) {




127,474,041✔





6749



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







6750



  }










6751



  if (acquired) {




127,471,063✔





6752



    (void)tsdbReleaseReader(pReader);




×







6753



  }










6754



  return code;




127,478,895✔





6755



}










6756












6757



static int32_t getBucketIndex(int32_t startRow, int32_t bucketRange, int32_t numOfRows, int32_t numOfBucket) {




1,156,517✔





6758



  if (numOfRows < startRow) {




1,156,517✔





6759



    return 0;




×







6760



  }










6761



  int32_t bucketIndex = ((numOfRows - startRow) / bucketRange);




1,156,517✔





6762



  if (bucketIndex == numOfBucket) {




1,156,517✔





6763



    bucketIndex -= 1;




×







6764



  }










6765



  return bucketIndex;




1,156,517✔





6766



}










6767












6768



void tsdbGetDataBlock(STsdbReader* pReader, SSDataBlock** pBlock) {




×







6769



  *pBlock = pReader->resBlockInfo.pResBlock;




×







6770



}




×







6771












6772



void tsdbSetDataBlock(STsdbReader* pReader, SSDataBlock* pBlock) {




408,201,289✔





6773



  pReader->resBlockInfo.pResBlock = pBlock;




408,201,289✔





6774



  if (pBlock) {




408,201,289✔





6775



    pReader->status.pPrimaryTsCol = taosArrayGet(pBlock->pDataBlock,  pReader->suppInfo.slotId[0]);




170,559,025✔





6776



  }










6777



}




408,201,289✔





6778












6779



int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pTableBlockInfo) {




116,110✔





6780



  int32_t       code = TSDB_CODE_SUCCESS;




116,110✔





6781



  int32_t       lino = 0;




116,110✔





6782



  const int32_t numOfBuckets = 20.0;




116,110✔





6783



  bool          acquired = false;




116,110✔





6784












6785



  TSDB_CHECK_NULL(pTableBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);




116,110✔





6786












6787



  pTableBlockInfo->totalSize = 0;




116,110✔





6788



  pTableBlockInfo->totalRows = 0;




120,357✔





6789



  pTableBlockInfo->numOfVgroups = 1;




120,357✔





6790












6791



  // find the start data block in file










6792



  code = tsdbAcquireReader(pReader);




120,357✔





6793



  TSDB_CHECK_CODE(code, lino, _end);




120,357✔





6794



  acquired = true;




120,357✔





6795












6796



  if (pReader->flag == READER_STATUS_SUSPEND) {




120,357✔





6797



    code = tsdbReaderResume2(pReader);




120,357✔





6798



    TSDB_CHECK_CODE(code, lino, _end);




120,357✔





6799



  }










6800












6801



  SMergeTreeConf conf = {




120,357✔





6802



      .pReader = pReader,










6803



      .pSchema = pReader->info.pSchema,




120,357✔





6804



      .pCols = pReader->suppInfo.colId,




120,357✔





6805



      .numOfCols = pReader->suppInfo.numOfCols,




120,357✔





6806



      .suid = pReader->info.suid,




120,357✔





6807



  };










6808












6809



  SReaderStatus* pStatus = &pReader->status;




120,357✔





6810



  if (pStatus->pCurrentFileset != NULL) {




120,357✔





6811



    pTableBlockInfo->numOfSttRows += tsdbGetRowsInSttFiles(pStatus->pCurrentFileset, pStatus->pLDataIterArray,




45,512✔





6812



                                                           pReader->pTsdb, &conf, pReader->idStr);




45,512✔





6813



  }










6814












6815



  STsdbCfg* pc = &pReader->pTsdb->pVnode->config.tsdbCfg;




120,357✔





6816



  pTableBlockInfo->defMinRows = pc->minRows;




120,357✔





6817



  pTableBlockInfo->defMaxRows = pc->maxRows;




120,357✔





6818












6819



  int32_t bucketRange = ceil(((double)(pc->maxRows - pc->minRows)) / numOfBuckets);




120,357✔





6820












6821



  pTableBlockInfo->numOfFiles += 1;




120,357✔





6822












6823



  int32_t numOfTables = (int32_t)tSimpleHashGetSize(pStatus->pTableMap);




120,357✔





6824












6825



  SDataBlockIter* pBlockIter = &pStatus->blockIter;




120,357✔





6826



  pTableBlockInfo->numOfFiles += pStatus->fileIter.numOfFiles;




120,357✔





6827












6828



  if (pBlockIter->numOfBlocks > 0) {




120,357✔





6829



    pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks;




20,708✔





6830



  }










6831












6832



  pTableBlockInfo->numOfTables = numOfTables;




120,357✔





6833



  bool hasNext = (pBlockIter->numOfBlocks > 0);




120,357✔





6834












6835



  while (true) {










6836



    if (hasNext) {




1,479,893✔





6837



      SFileDataBlockInfo* pBlockInfo = NULL;




1,156,517✔





6838



      code = getCurrentBlockInfo(pBlockIter, &pBlockInfo, pReader->idStr);




1,156,517✔





6839



      TSDB_CHECK_CODE(code, lino, _end);




1,156,517✔





6840












6841



      int32_t numOfRows = pBlockInfo->numRow;




1,156,517✔





6842












6843



      pTableBlockInfo->totalRows += numOfRows;




1,156,517✔





6844












6845



      if (numOfRows > pTableBlockInfo->maxRows) {




1,156,517✔





6846



        pTableBlockInfo->maxRows = numOfRows;




37,169✔





6847



      }










6848












6849



      if (numOfRows < pTableBlockInfo->minRows) {




1,156,517✔





6850



        pTableBlockInfo->minRows = numOfRows;




37,169✔





6851



      }










6852












6853



      pTableBlockInfo->totalSize += pBlockInfo->blockSize;




1,156,517✔





6854












6855



      int32_t bucketIndex = getBucketIndex(pTableBlockInfo->defMinRows, bucketRange, numOfRows, numOfBuckets);




1,156,517✔





6856



      pTableBlockInfo->blockRowsHisto[bucketIndex]++;




1,156,517✔





6857












6858



      hasNext = blockIteratorNext(&pStatus->blockIter);




1,156,517✔





6859



    } else {










6860



      code = initForFirstBlockInFile(pReader, pBlockIter);




323,376✔





6861



      TSDB_CHECK_CODE(code, lino, _end);




323,376✔





6862



      if (pStatus->loadFromFile == false) {




323,376✔





6863



        break;




120,357✔





6864



      }










6865












6866



      // add the data in stt files of new fileset










6867



      if (pStatus->pCurrentFileset != NULL) {




203,019✔





6868



        pTableBlockInfo->numOfSttRows += tsdbGetRowsInSttFiles(pStatus->pCurrentFileset, pStatus->pLDataIterArray,




203,019✔





6869



                                                               pReader->pTsdb, &conf, pReader->idStr);




203,019✔





6870



      }










6871












6872



      pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks;




203,019✔





6873



      hasNext = (pBlockIter->numOfBlocks > 0);




203,019✔





6874



    }










6875



  }










6876












6877



  // record the data in stt files










6878



_end:




120,357✔





6879



  if (code != TSDB_CODE_SUCCESS) {




120,357✔





6880



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







6881



  }










6882



  if (acquired) {




120,357✔





6883



    (void)tsdbReleaseReader(pReader);




120,357✔





6884



  }










6885



  return code;




120,357✔





6886



}










6887












6888



static void getMemTableTimeRange(STsdbReader* pReader, int64_t* pMaxKey, int64_t* pMinKey) {




293,911,559✔





6889



  SReaderStatus* pStatus = &pReader->status;




293,911,559✔





6890












6891



  int32_t iter = 0;




293,934,384✔





6892



  int64_t maxKey = INT64_MIN;




293,941,228✔





6893



  int64_t minKey = INT64_MAX;




293,941,228✔





6894












6895



  void* pHashIter = tSimpleHashIterate(pStatus->pTableMap, NULL, &iter);




293,941,228✔





6896



  while (pHashIter != NULL) {




1,677,509,694✔





6897



    STableBlockScanInfo* pBlockScanInfo = *(STableBlockScanInfo**)pHashIter;




1,383,534,857✔





6898












6899



    STbData* d = NULL;




1,383,458,715✔





6900



    if (pReader->pReadSnap->pMem != NULL) {




1,383,458,715✔





6901



      d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->info.suid, pBlockScanInfo->uid);




1,221,891,276✔





6902



      if (d != NULL) {




1,221,979,979✔





6903



        if (d->maxKey > maxKey) {




1,133,318,780✔





6904



          maxKey = d->maxKey;




202,553,548✔





6905



        }










6906



        if (d->minKey < minKey) {




1,133,300,578✔





6907



          minKey = d->minKey;




192,710,053✔





6908



        }










6909



      }










6910



    }










6911












6912



    STbData* di = NULL;




1,383,614,716✔





6913



    if (pReader->pReadSnap->pIMem != NULL) {




1,383,614,716✔





6914



      di = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->info.suid, pBlockScanInfo->uid);




4,870✔





6915



      if (di != NULL) {




4,870✔





6916



        if (di->maxKey > maxKey) {




4,870✔





6917



          maxKey = di->maxKey;




4,870✔





6918



        }










6919



        if (di->minKey < minKey) {




4,870✔





6920



          minKey = di->minKey;




4,870✔





6921



        }










6922



      }










6923



    }










6924












6925



    // current table is exhausted, let's try the next table










6926



    pHashIter = tSimpleHashIterate(pStatus->pTableMap, pHashIter, &iter);




1,383,623,470✔





6927



  }










6928












6929



  *pMaxKey = maxKey;




293,974,837✔





6930



  *pMinKey = minKey;




293,973,270✔





6931



}




293,969,772✔





6932












6933



int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader* pReader, uint32_t* rows) {




120,357✔





6934



  int32_t        code = TSDB_CODE_SUCCESS;




120,357✔





6935



  int32_t        lino = 0;




120,357✔





6936



  SReaderStatus* pStatus = NULL;




120,357✔





6937



  bool           acquired = false;




120,357✔





6938












6939



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




120,357✔





6940



  TSDB_CHECK_NULL(rows, code, lino, _end, TSDB_CODE_INVALID_PARA);




120,357✔





6941












6942



  *rows = 0;




120,357✔





6943



  pStatus = &pReader->status;




120,357✔





6944



  code = tsdbAcquireReader(pReader);




120,357✔





6945



  TSDB_CHECK_CODE(code, lino, _end);




120,357✔





6946



  acquired = true;




120,357✔





6947












6948



  if (pReader->flag == READER_STATUS_SUSPEND) {




120,357✔





6949



    code = tsdbReaderResume2(pReader);




32,716✔





6950



    TSDB_CHECK_CODE(code, lino, _end);




32,716✔





6951



  }










6952












6953



  int32_t iter = 0;




120,357✔





6954



  pStatus->pTableIter = tSimpleHashIterate(pStatus->pTableMap, NULL, &iter);




120,357✔





6955












6956



  while (pStatus->pTableIter != NULL) {




251,497✔





6957



    STableBlockScanInfo* pBlockScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter;




131,140✔





6958












6959



    STbData* d = NULL;




131,140✔





6960



    if (pReader->pReadSnap->pMem != NULL) {




131,140✔





6961



      d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->info.suid, pBlockScanInfo->uid);




57,741✔





6962



      if (d != NULL) {




57,741✔





6963



        *rows += tsdbGetNRowsInTbData(d);




46,767✔





6964



      }










6965



    }










6966












6967



    STbData* di = NULL;




131,140✔





6968



    if (pReader->pReadSnap->pIMem != NULL) {




131,140✔





6969



      di = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->info.suid, pBlockScanInfo->uid);




22,003✔





6970



      if (di != NULL) {




22,003✔





6971



        *rows += tsdbGetNRowsInTbData(di);




22,003✔





6972



      }










6973



    }










6974












6975



    // current table is exhausted, let's try the next table










6976



    pStatus->pTableIter = tSimpleHashIterate(pStatus->pTableMap, pStatus->pTableIter, &iter);




131,140✔





6977



  }










6978












6979



_end:




120,357✔





6980



  if (code != TSDB_CODE_SUCCESS) {




120,357✔





6981



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







6982



  }










6983



  if (acquired) {




120,357✔





6984



    (void)tsdbReleaseReader(pReader);




120,357✔





6985



  }










6986



  return code;




120,357✔





6987



}










6988












6989



int32_t tsdbGetTableSchema(SMeta* pMeta, int64_t uid, STSchema** pSchema, int64_t* suid, SSchemaWrapper** pTagSchema) {




541,689✔





6990



  SMetaReader mr = {0};




541,689✔





6991



  metaReaderDoInit(&mr, pMeta, META_READER_LOCK);




541,689✔





6992



  int32_t code = metaReaderGetTableEntryByUidCache(&mr, uid);




541,689✔





6993



  if (code != TSDB_CODE_SUCCESS) {




541,689✔





6994



    code = TSDB_CODE_TDB_INVALID_TABLE_ID;




×







6995



    metaReaderClear(&mr);




×







6996



    return code;




×







6997



  }










6998












6999



  *suid = 0;




541,689✔





7000












7001



  if (mr.me.type == TSDB_CHILD_TABLE) {




541,689✔





7002



    tDecoderClear(&mr.coder);




200,734✔





7003



    *suid = mr.me.ctbEntry.suid;




200,734✔





7004



    code = metaReaderGetTableEntryByUidCache(&mr, *suid);




200,734✔





7005



    if (code != TSDB_CODE_SUCCESS) {




200,734✔





7006



      code = TSDB_CODE_TDB_INVALID_TABLE_ID;




×







7007



      metaReaderClear(&mr);




×







7008



      return code;




×







7009



    }










7010



  } else if (mr.me.type == TSDB_NORMAL_TABLE) {  // do nothing




340,955✔





7011



  } else if (mr.me.type == TSDB_SUPER_TABLE) {




16,340✔





7012



    tDecoderClear(&mr.coder);




16,340✔





7013



    *suid = uid;




16,340✔





7014



    code = metaReaderGetTableEntryByUidCache(&mr, *suid);




16,340✔





7015



    if (code != TSDB_CODE_SUCCESS) {




16,340✔





7016



      code = TSDB_CODE_TDB_INVALID_TABLE_ID;




×







7017



      metaReaderClear(&mr);




×







7018



      return code;




×







7019



    }










7020



    *pTagSchema = tCloneSSchemaWrapper(&mr.me.stbEntry.schemaTag);




16,340✔





7021



  } else {










7022



    code = TSDB_CODE_INVALID_PARA;




×







7023



    tsdbError("invalid mr.me.type:%d, code:%s", mr.me.type, tstrerror(code));




×







7024



    metaReaderClear(&mr);




×







7025



    return code;




×







7026



  }










7027












7028



  metaReaderClear(&mr);




541,689✔





7029












7030



  // get the newest table schema version










7031



  code = metaGetTbTSchemaEx(pMeta, *suid, uid, -1, pSchema);




541,689✔





7032



  return code;




541,689✔





7033



}










7034












7035



int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STsdbReadSnap** ppSnap, const char* id) {




1,690,040,145✔





7036



  int32_t        code = 0;




1,690,040,145✔





7037



  STsdb*         pTsdb = pReader->pTsdb;




1,690,040,145✔





7038



  SVersionRange* pRange = &pReader->info.verRange;




1,690,465,945✔





7039



  int32_t        lino = 0;




1,690,175,796✔





7040



  *ppSnap = NULL;




1,690,175,796✔





7041












7042



  // lock










7043



  code = taosThreadMutexLock(&pTsdb->mutex);




1,690,020,809✔





7044



  if (code != TSDB_CODE_SUCCESS) {




1,690,521,657✔





7045



    tsdbError("%s failed to lock tsdb, code:%s", id, tstrerror(code));




×







7046



    return code;




×







7047



  }










7048












7049



  // alloc










7050



  STsdbReadSnap* pSnap = (STsdbReadSnap*)taosMemoryCalloc(1, sizeof(STsdbReadSnap));




1,690,521,657✔





7051



  if (pSnap == NULL) {




1,690,383,174✔





7052



    (void)taosThreadMutexUnlock(&pTsdb->mutex);




×







7053



    TSDB_CHECK_NULL(pSnap, code, lino, _exit, terrno);




×







7054



  }










7055












7056



  // take snapshot










7057



  if (pTsdb->mem && (pRange->minVer <= pTsdb->mem->maxVer && pRange->maxVer >= pTsdb->mem->minVer)) {




1,690,383,174✔





7058



    pSnap->pMem = pTsdb->mem;




1,290,445,469✔





7059



    pSnap->pNode = taosMemoryMalloc(sizeof(*pSnap->pNode));




1,290,757,622✔





7060



    if (pSnap->pNode == NULL) {




1,290,078,562✔





7061



      (void)taosThreadMutexUnlock(&pTsdb->mutex);




×







7062



      TSDB_CHECK_NULL(pSnap->pNode, code, lino, _exit, terrno);




×







7063



    }










7064












7065



    pSnap->pNode->pQHandle = pReader;




1,290,147,328✔





7066



    pSnap->pNode->reseek = reseek;




1,290,480,049✔





7067












7068



    code = tsdbRefMemTable(pTsdb->mem, pSnap->pNode);




1,290,432,293✔





7069



    TSDB_CHECK_CODE(code, lino, _exit);




1,290,614,407✔





7070



  }










7071












7072



  if (pTsdb->imem && (pRange->minVer <= pTsdb->imem->maxVer && pRange->maxVer >= pTsdb->imem->minVer)) {




1,690,176,859✔





7073



    pSnap->pIMem = pTsdb->imem;




8,854,260✔





7074



    pSnap->pINode = taosMemoryMalloc(sizeof(*pSnap->pINode));




8,854,260✔





7075



    if (pSnap->pINode == NULL) {




8,853,942✔





7076



      code = terrno;




×







7077












7078



      if (pTsdb->mem && pSnap->pNode) {




×







7079



        tsdbUnrefMemTable(pTsdb->mem, pSnap->pNode, true);  // unref the previous refed mem




×







7080



      }










7081












7082



      (void)taosThreadMutexUnlock(&pTsdb->mutex);




×







7083



      goto _exit;




×







7084



    }










7085












7086



    pSnap->pINode->pQHandle = pReader;




8,851,969✔





7087



    pSnap->pINode->reseek = reseek;




8,852,287✔





7088












7089



    code = tsdbRefMemTable(pTsdb->imem, pSnap->pINode);




8,851,933✔





7090



    TSDB_CHECK_CODE(code, lino, _exit);




8,852,287✔





7091



  }










7092












7093



  // fs










7094



  code = tsdbFSCreateRefSnapshotWithoutLock(pTsdb->pFS, &pSnap->pfSetArray);




1,690,648,474✔





7095



  if (code) {




1,690,625,218✔





7096



    if (pSnap->pNode) {




×







7097



      tsdbUnrefMemTable(pTsdb->mem, pSnap->pNode, true);  // unref the previous refed mem




×







7098



    }










7099












7100



    if (pSnap->pINode) {




×







7101



      tsdbUnrefMemTable(pTsdb->imem, pSnap->pINode, true);




×







7102



    }










7103












7104



    (void)taosThreadMutexUnlock(&pTsdb->mutex);




×







7105



    TSDB_CHECK_CODE(code, lino, _exit);




×







7106



  }










7107












7108



  // unlock










7109



  (void)taosThreadMutexUnlock(&pTsdb->mutex);




1,690,625,218✔





7110



  *ppSnap = pSnap;




1,691,115,072✔





7111












7112



  tsdbTrace("%s vgId:%d, take read snapshot", id, TD_VID(pTsdb->pVnode));




1,691,119,254✔





7113



  return code;




1,690,477,747✔





7114












7115



_exit:




×







7116



  tsdbError("%s vgId:%d take read snapshot failed, line:%d code:%s", id, TD_VID(pTsdb->pVnode), lino, tstrerror(code));




×







7117












7118



  if (pSnap) {




×







7119



    if (pSnap->pNode) taosMemoryFree(pSnap->pNode);




×







7120



    if (pSnap->pINode) taosMemoryFree(pSnap->pINode);




×







7121



    taosMemoryFree(pSnap);




×







7122



  }










7123



  return code;




×







7124



}










7125












7126



void tsdbUntakeReadSnap2(STsdbReader* pReader, STsdbReadSnap* pSnap, bool proactive) {




1,690,868,178✔





7127



  STsdb* pTsdb = pReader->pTsdb;




1,690,868,178✔





7128












7129



  if (pSnap) {




1,690,940,590✔





7130



    if (pSnap->pMem) {




1,691,011,495✔





7131



      tsdbUnrefMemTable(pSnap->pMem, pSnap->pNode, proactive);




1,290,860,293✔





7132



    }










7133












7134



    if (pSnap->pIMem) {




1,691,007,629✔





7135



      tsdbUnrefMemTable(pSnap->pIMem, pSnap->pINode, proactive);




8,854,260✔





7136



    }










7137












7138



    if (pSnap->pNode) taosMemoryFree(pSnap->pNode);




1,691,123,973✔





7139



    if (pSnap->pINode) taosMemoryFree(pSnap->pINode);




1,690,891,515✔





7140












7141



    tsdbFSDestroyRefSnapshot(&pSnap->pfSetArray);




1,690,929,584✔





7142












7143



    taosMemoryFree(pSnap);




1,690,815,009✔





7144



  }










7145



  tsdbTrace("vgId:%d, untake read snapshot", TD_VID(pTsdb->pVnode));




1,690,449,079✔





7146



}




1,690,449,646✔





7147












7148



// if failed, do nothing










7149



int32_t tsdbReaderSetId(void* p, const char* idstr) {




2,506,377✔





7150



  int32_t      code = TSDB_CODE_SUCCESS;




2,506,377✔





7151



  int32_t      lino = 0;




2,506,377✔





7152



  STsdbReader* pReader = (STsdbReader*)p;




2,506,377✔





7153












7154



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);




2,506,377✔





7155












7156



  taosMemoryFreeClear(pReader->idStr);




2,506,377✔





7157



  pReader->idStr = taosStrdup(idstr);




2,506,033✔





7158



  TSDB_CHECK_NULL(pReader->idStr, code, lino, _end, terrno);




2,508,279✔





7159












7160



  pReader->status.fileIter.pSttBlockReader->mergeTree.idStr = pReader->idStr;




2,509,359✔





7161












7162



_end:




2,505,334✔





7163



  if (code != TSDB_CODE_SUCCESS) {




2,505,334✔





7164



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));




×







7165



  }










7166



  return code;




2,505,334✔





7167



}










7168












7169



void tsdbReaderSetCloseFlag(STsdbReader* pReader) { /*pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED;*/




239,502✔





7170



}




239,502✔





7171












7172



void tsdbSetFilesetDelimited(STsdbReader* pReader) { pReader->bFilesetDelimited = true; }




293,799,763✔





7173












7174



void tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn notifyFn, void* param) {




290,587,593✔





7175



  pReader->notifyFn = notifyFn;




290,587,593✔





7176



  pReader->notifyParam = param;




290,671,201✔





7177



}




290,574,680✔





7178












7179



#if 0










7180



static int32_t tsdbEncodeLastProcKeys(const SSHashObj* pTableMap, void** buf) {










7181



  void* p = NULL;










7182



  int32_t iter = 0;










7183



  int32_t len = 0;










7184












7185



  len += taosEncodeFixedI32(buf, tSimpleHashGetSize(pTableMap)); // number of tables










7186












7187



  while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) {










7188



    uint64_t uid = *(uint64_t*)tSimpleHashGetKey(p, NULL);










7189



    SRowKey* pKey = &(*(STableBlockScanInfo**)p)->lastProcKey;










7190












7191



    len += taosEncodeFixedU64(buf, uid);            // table uid










7192



    len += taosEncodeFixedI64(buf, pKey->ts);       // last read ts










7193



    len += taosEncodeFixedU8(buf, pKey->numOfPKs);  // number of pks










7194












7195



    if (pKey->numOfPKs > 0) {










7196



      int8_t type = pKey->pks[0].type;










7197



      len += taosEncodeFixedI8(buf, type);  // pk data type










7198












7199



      uint8_t* pData = 0;










7200



      uint32_t nData = 0;










7201



      if (IS_VAR_DATA_TYPE(type)) {










7202



        pData = pKey->pks[0].pData;










7203



        nData = pKey->pks[0].nData;










7204



        len += taosEncodeFixedU32(buf, nData);  // data length










7205



      } else {










7206



        pData = (uint8_t*)&pKey->pks[0].val;










7207



        nData = tDataTypes[pKey->pks[0].type].bytes;










7208



      }










7209



      len += taosEncodeBinary(buf, pData, nData); // data value










7210



    }










7211



  }










7212












7213



  return len;










7214



}










7215












7216



static int32_t tsdbDecodeLastProcKeys(const void* buf, int32_t len, SSHashObj* pTableMap) {










7217



  int32_t code = TSDB_CODE_SUCCESS;










7218



  int32_t lino = 0;










7219



  const void *begin = buf;










7220












7221



  TSDB_CHECK_NULL(buf, code, lino, _end, TSDB_CODE_INVALID_PARA);










7222



  TSDB_CHECK_NULL(pTableMap, code, lino, _end, TSDB_CODE_INVALID_PARA);










7223












7224



  int32_t numOfTables = 0;










7225



  buf = taosDecodeFixedI32(buf, &numOfTables);










7226












7227



  TSDB_CHECK_CONDITION(numOfTables == tSimpleHashGetSize(pTableMap), code, lino, _end, TSDB_CODE_INVALID_PARA);










7228












7229



  for (int32_t i = 0; i < numOfTables; ++i) {










7230



    uint64_t uid = 0;










7231



    buf = taosDecodeFixedU64(buf, &uid);










7232












7233



    STableBlockScanInfo** pScanInfo = tSimpleHashGet(pTableMap, &uid, sizeof(uid));










7234



    if (pScanInfo == NULL) {










7235



      tsdbError("failed to find scan info of uid: %" PRIu64 " when decoding last proc keys", uid);










7236



      TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);










7237



    }










7238



    SRowKey* pKey = &(*pScanInfo)->lastProcKey;










7239












7240



    buf = taosDecodeFixedI64(buf, &pKey->ts);










7241



    uint8_t numOfPks = 0;










7242



    buf = taosDecodeFixedU8(buf, &numOfPks);










7243



    if (numOfPks != pKey->numOfPKs) {










7244



      tsdbError("numOfPks of uid %" PRIu64 " mismatch: [%u/%u]", uid, numOfPks, pKey->numOfPKs);










7245



      TSDB_CHECK_CONDITION(numOfPks == pKey->numOfPKs, code, lino, _end, TSDB_CODE_INVALID_PARA);










7246



    }










7247












7248



    if (numOfPks > 0) {










7249



      int8_t type = 0;










7250



      buf = taosDecodeFixedI8(buf, &type);










7251



      if (type != pKey->pks[0].type) {










7252



        tsdbError("pk type of uid %" PRIu64 " mismatch: [%d/%d]", uid, type, pKey->pks[0].type);










7253



        TSDB_CHECK_CONDITION(type == pKey->pks[0].type, code, lino, _end, TSDB_CODE_INVALID_PARA);










7254



      }










7255












7256



      uint8_t *pData = 0;










7257



      uint32_t nData = 0;










7258



      if (IS_VAR_DATA_TYPE(type)) {










7259



        pData = pKey->pks[0].pData;










7260



        buf = taosDecodeFixedU32(buf, &nData);










7261



        pKey->pks[0].nData = nData;










7262



      } else {










7263



        pKey->pks[0].val = 0;










7264



        pData = (uint8_t*)&pKey->pks[0].val;










7265



        nData = tDataTypes[type].bytes;










7266



      }










7267



      buf = taosDecodeBinary(buf, (void **)&pData, nData);










7268



    }










7269



  }










7270












7271



  TSDB_CHECK_CONDITION(buf == begin + len , code, lino, _end, TSDB_CODE_INVALID_PARA);










7272












7273



_end:










7274



  if (code != TSDB_CODE_SUCCESS) {










7275



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));










7276



  }










7277



  return code;










7278



}










7279



#endif










7280












7281



int32_t tsdbReaderGetProgress(const STsdbReader* pReader, void** pBuf, uint64_t* pLen) {




×







7282



  int32_t code = TSDB_CODE_SUCCESS;




×







7283



#if 0










7284



  int32_t              lino = 0;










7285



  void*                buf = NULL;










7286



  int32_t len = 0;










7287












7288



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);










7289



  TSDB_CHECK_NULL(pBuf, code, lino, _end, TSDB_CODE_INVALID_PARA);










7290



  TSDB_CHECK_NULL(pLen, code, lino, _end, TSDB_CODE_INVALID_PARA);










7291












7292



  *pBuf = NULL;










7293



  *pLen = tsdbEncodeLastProcKeys(pReader->status.pTableMap, NULL);










7294












7295



  buf = taosMemoryMalloc(*pLen);










7296



  TSDB_CHECK_NULL(buf, code, lino, _end, terrno);










7297



  *pBuf = buf;










7298



  len = tsdbEncodeLastProcKeys(pReader->status.pTableMap, &buf);










7299



  TSDB_CHECK_CONDITION(len == *pLen, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);










7300



  buf = NULL;










7301












7302



_end:










7303



  if (code != TSDB_CODE_SUCCESS) {










7304



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));










7305



  }










7306



  if (buf != NULL) {










7307



    taosMemoryFreeClear(buf);










7308



  }










7309



#endif










7310



  return code;




×







7311



}










7312












7313



int32_t tsdbReaderSetProgress(STsdbReader* pReader, const void* buf, uint64_t len) {




×







7314



  int32_t code = TSDB_CODE_SUCCESS;




×







7315



#if 0










7316



  int32_t lino = 0;










7317












7318



  if (len == 0) {










7319



    goto _end;










7320



  }










7321












7322



  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);










7323



  TSDB_CHECK_NULL(buf, code, lino, _end, TSDB_CODE_INVALID_PARA);










7324












7325



  code = tsdbDecodeLastProcKeys(buf, len, pReader->status.pTableMap);










7326



  TSDB_CHECK_CODE(code, lino, _end);










7327












7328



_end:










7329



  if (code != TSDB_CODE_SUCCESS) {










7330



    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));










7331



  }










7332



#endif










7333



  return code;




×







7334



}
























    

  • 

    •