taosdata/TDengine | Build 3780 | source/libs/executor/src/streamtimewindowoperator.c | Coveralls

50

int32_t binarySearchCom(void* keyList, int num, void* pKey, int order, __compare_fn_t comparefn) {

×

51

  int firstPos = 0, lastPos = num - 1, midPos = -1;

×

52

  int numOfRows = 0;

×

54

  if (num <= 0) return -1;

×

55

  if (order == TSDB_ORDER_DESC) {

×

58

      if (comparefn(pKey, keyList, lastPos) >= 0) return lastPos;

×

59

      if (comparefn(pKey, keyList, firstPos) == 0) return firstPos;

×

60

      if (comparefn(pKey, keyList, firstPos) < 0) return firstPos - 1;

×

62

      numOfRows = lastPos - firstPos + 1;

×

63

      midPos = (numOfRows >> 1) + firstPos;

×

65

      if (comparefn(pKey, keyList, midPos) < 0) {

×

66

        lastPos = midPos - 1;

×

67

      } else if (comparefn(pKey, keyList, midPos) > 0) {

×

68

        firstPos = midPos + 1;

×

70

        break;

×

77

      if (comparefn(pKey, keyList, firstPos) <= 0) return firstPos;

×

78

      if (comparefn(pKey, keyList, lastPos) == 0) return lastPos;

×

80

      if (comparefn(pKey, keyList, lastPos) > 0) {

×

81

        lastPos = lastPos + 1;

×

82

        if (lastPos >= num)

×

83

          return -1;

×

85

          return lastPos;

×

88

      numOfRows = lastPos - firstPos + 1;

×

89

      midPos = (numOfRows >> 1) + firstPos;

×

91

      if (comparefn(pKey, keyList, midPos) < 0) {

×

92

        lastPos = midPos - 1;

×

93

      } else if (comparefn(pKey, keyList, midPos) > 0) {

×

94

        firstPos = midPos + 1;

×

96

        break;

×

101

  return midPos;

×

104

static int32_t comparePullWinKey(void* pKey, void* data, int32_t index) {

×

105

  SArray*          res = (SArray*)data;

×

106

  SPullWindowInfo* pos = taosArrayGet(res, index);

×

107

  SPullWindowInfo* pData = (SPullWindowInfo*)pKey;

×

108

  if (pData->groupId > pos->groupId) {

×

109

    return 1;

×

110

  } else if (pData->groupId < pos->groupId) {

×

111

    return -1;

×

114

  if (pData->window.skey > pos->window.ekey) {

×

115

    return 1;

×

116

  } else if (pData->window.ekey < pos->window.skey) {

×

117

    return -1;

×

119

  return 0;

×

122

static int32_t savePullWindow(SPullWindowInfo* pPullInfo, SArray* pPullWins) {

×

123

  int32_t size = taosArrayGetSize(pPullWins);

×

124

  int32_t index = binarySearchCom(pPullWins, size, pPullInfo, TSDB_ORDER_DESC, comparePullWinKey);

×

125

  if (index == -1) {

×

126

    index = 0;

×

128

    int32_t code = comparePullWinKey(pPullInfo, pPullWins, index);

×

129

    if (code == 0) {

×

130

      SPullWindowInfo* pos = taosArrayGet(pPullWins, index);

×

131

      pos->window.skey = TMIN(pos->window.skey, pPullInfo->window.skey);

×

132

      pos->window.ekey = TMAX(pos->window.ekey, pPullInfo->window.ekey);

×

133

      pos->calWin.skey = TMIN(pos->calWin.skey, pPullInfo->calWin.skey);

×

134

      pos->calWin.ekey = TMAX(pos->calWin.ekey, pPullInfo->calWin.ekey);

×

135

      return TSDB_CODE_SUCCESS;

×

136

    } else if (code > 0) {

×

137

      index++;

×

140

  if (taosArrayInsert(pPullWins, index, pPullInfo) == NULL) {

×

141

    return terrno;

×

143

  return TSDB_CODE_SUCCESS;

×

146

int32_t saveResult(SResultWindowInfo winInfo, SSHashObj* pStUpdated) {

×

147

  if (tSimpleHashGetSize(pStUpdated) > MAX_STREAM_HISTORY_RESULT) {

×

148

    qError("%s failed at line %d since too many history result. ", __func__, __LINE__);

×

149

    return TSDB_CODE_STREAM_INTERNAL_ERROR;

×

151

  winInfo.sessionWin.win.ekey = winInfo.sessionWin.win.skey;

×

152

  return tSimpleHashPut(pStUpdated, &winInfo.sessionWin, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo));

×

155

int32_t saveWinResult(SWinKey* pKey, SRowBuffPos* pPos, SSHashObj* pUpdatedMap) {

×

156

  if (tSimpleHashGetSize(pUpdatedMap) > MAX_STREAM_HISTORY_RESULT) {

×

157

    qError("%s failed at line %d since too many history result. ", __func__, __LINE__);

×

158

    return TSDB_CODE_STREAM_INTERNAL_ERROR;

×

160

  return tSimpleHashPut(pUpdatedMap, pKey, sizeof(SWinKey), &pPos, POINTER_BYTES);

×

163

static int32_t saveWinResultInfo(TSKEY ts, uint64_t groupId, SRowBuffPos* pPos, SSHashObj* pUpdatedMap) {

×

164

  SWinKey key = {.ts = ts, .groupId = groupId};

×

165

  return saveWinResult(&key, pPos, pUpdatedMap);

×

168

static void removeResults(SArray* pWins, SSHashObj* pUpdatedMap) {

×

169

  int32_t size = taosArrayGetSize(pWins);

×

170

  for (int32_t i = 0; i < size; i++) {

×

171

    SWinKey* pW = taosArrayGet(pWins, i);

×

172

    void*    tmp = tSimpleHashGet(pUpdatedMap, pW, sizeof(SWinKey));

×

173

    if (tmp) {

×

174

      void* value = *(void**)tmp;

×

175

      taosMemoryFree(value);

×

176

      int32_t tmpRes = tSimpleHashRemove(pUpdatedMap, pW, sizeof(SWinKey));

×

177

      qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

180

×

182

int32_t compareWinKey(void* pKey, void* data, int32_t index) {

×

183

  void* pDataPos = taosArrayGet((SArray*)data, index);

×

184

  return winKeyCmprImpl(pKey, pDataPos);

×

187

void removeDeleteResults(SSHashObj* pUpdatedMap, SArray* pDelWins) {

×

188

  taosArraySort(pDelWins, winKeyCmprImpl);

×

189

  taosArrayRemoveDuplicate(pDelWins, winKeyCmprImpl, NULL);

×

190

  int32_t delSize = taosArrayGetSize(pDelWins);

×

191

  if (tSimpleHashGetSize(pUpdatedMap) == 0 || delSize == 0) {

×

192

    return;

×

194

  void*   pIte = NULL;

×

195

  int32_t iter = 0;

×

196

  while ((pIte = tSimpleHashIterate(pUpdatedMap, pIte, &iter)) != NULL) {

×

197

    SWinKey* pResKey = tSimpleHashGetKey(pIte, NULL);

×

198

    int32_t  index = binarySearchCom(pDelWins, delSize, pResKey, TSDB_ORDER_DESC, compareWinKey);

×

199

    if (index >= 0 && 0 == compareWinKey(pResKey, pDelWins, index)) {

×

200

      taosArrayRemove(pDelWins, index);

×

201

      delSize = taosArrayGetSize(pDelWins);

×

206

bool isOverdue(TSKEY ekey, STimeWindowAggSupp* pTwSup) {

×

207

  return pTwSup->maxTs != INT64_MIN && ekey < pTwSup->maxTs - pTwSup->waterMark;

×

210

bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pTwSup) { return isOverdue(pWin->ekey, pTwSup); }

×

212

static void doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId) {

×

213

  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;

×

215

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

216

  SWinKey                      key = {.ts = ts, .groupId = groupId};

×

217

  int32_t                      tmpRes = tSimpleHashRemove(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey));

×

218

  qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

219

  pAPI->stateStore.streamStateDel(pInfo->pState, &key);

×

220

×

222

int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; }

×

224

static void doDeleteWindowByGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock) {

×

225

  SStorageAPI*                 pAPI = &pOperator->pTaskInfo->storageAPI;

×

226

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

228

  SColumnInfoData* pGpIdCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);

×

229

  uint64_t* pGroupIdData = (uint64_t*)pGpIdCol->pData;

×

230

  for (int32_t i = 0; i < pBlock->info.rows; i++) {

×

231

    uint64_t groupId = pGroupIdData[i];

×

232

    void*   pIte = NULL;

×

233

    int32_t iter = 0;

×

234

    while ((pIte = tSimpleHashIterate(pInfo->aggSup.pResultRowHashTable, pIte, &iter)) != NULL) {

×

235

      size_t keyLen = 0;

×

236

      SWinKey* pKey = tSimpleHashGetKey(pIte, &keyLen);

×

237

      if (pKey->groupId == groupId) {

×

238

        int32_t tmpRes = tSimpleHashIterateRemove(pInfo->aggSup.pResultRowHashTable, pKey, keyLen, &pIte, &iter);

×

239

        qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

243

    pAPI->stateStore.streamStateDelByGroupId(pInfo->pState, groupId);

×

245

×

247

static int32_t doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDataBlock* pBlock, SArray* pUpWins,

×

249

  int32_t                      code = TSDB_CODE_SUCCESS;

×

250

  int32_t                      lino = 0;

×

251

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

252

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

253

  SColumnInfoData*             pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);

×

254

  TSKEY*                       startTsCols = (TSKEY*)pStartTsCol->pData;

×

255

  SColumnInfoData*             pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);

×

256

  TSKEY*                       endTsCols = (TSKEY*)pEndTsCol->pData;

×

257

  SColumnInfoData*             pCalStTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);

×

258

  TSKEY*                       calStTsCols = (TSKEY*)pCalStTsCol->pData;

×

259

  SColumnInfoData*             pCalEnTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);

×

260

  TSKEY*                       calEnTsCols = (TSKEY*)pCalEnTsCol->pData;

×

261

  SColumnInfoData*             pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);

×

262

  uint64_t*                    pGpDatas = (uint64_t*)pGpCol->pData;

×

263

  for (int32_t i = 0; i < pBlock->info.rows; i++) {

×

264

    SResultRowInfo dumyInfo = {0};

×

265

    dumyInfo.cur.pageId = -1;

×

267

    STimeWindow win = {0};

×

268

    if (IS_FINAL_INTERVAL_OP(pOperator) || IS_MID_INTERVAL_OP(pOperator)) {

×

269

      win.skey = startTsCols[i];

×

270

      win.ekey = endTsCols[i];

×

272

      win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC);

×

276

      if (!inCalSlidingWindow(pInterval, &win, calStTsCols[i], calEnTsCols[i], pBlock->info.type)) {

×

277

        getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC);

×

278

        continue;

×

280

      uint64_t winGpId = pGpDatas[i];

×

281

      SWinKey  winRes = {.ts = win.skey, .groupId = winGpId};

×

282

      void*    chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey));

×

283

      if (chIds) {

×

284

        int32_t childId = getChildIndex(pBlock);

×

285

        if (pInvalidWins) {

×

286

          qDebug("===stream===save invalid delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d", winRes.ts,

×

288

          code = taosHashPut(pInvalidWins, &winRes, sizeof(SWinKey), NULL, 0);

×

289

          QUERY_CHECK_CODE(code, lino, _end);

×

292

        SArray* chArray = *(void**)chIds;

×

293

        int32_t index = taosArraySearchIdx(chArray, &childId, compareInt32Val, TD_EQ);

×

294

        if (index != -1) {

×

295

          qDebug("===stream===try push delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d ,continue", win.skey,

×

297

          getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC);

×

298

          continue;

×

301

      doDeleteWindow(pOperator, win.skey, winGpId);

×

302

      if (pUpWins) {

×

303

        void* tmp = taosArrayPush(pUpWins, &winRes);

×

304

        if (!tmp) {

×

305

          code = terrno;

×

306

          QUERY_CHECK_CODE(code, lino, _end);

×

309

      if (pUpdatedMap) {

×

310

        int32_t tmpRes = tSimpleHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey));

×

311

        qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

313

      getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC);

×

314

    } while (win.ekey <= endTsCols[i]);

×

316

_end:

×

317

  if (code != TSDB_CODE_SUCCESS) {

×

318

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

320

  return code;

×

323

static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SSHashObj* resWins) {

×

324

  void*   pIte = NULL;

×

325

  int32_t iter = 0;

×

326

  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {

×

327

    SWinKey*     pKey = tSimpleHashGetKey(pIte, NULL);

×

328

    uint64_t     groupId = pKey->groupId;

×

329

    TSKEY        ts = pKey->ts;

×

330

    SRowBuffPos* pPos = *(SRowBuffPos**)pIte;

×

331

    if (!pPos->beUpdated) {

×

332

      continue;

×

334

    pPos->beUpdated = false;

×

335

    int32_t code = saveWinResultInfo(ts, groupId, pPos, resWins);

×

336

    if (code != TSDB_CODE_SUCCESS) {

×

337

      return code;

×

340

  return TSDB_CODE_SUCCESS;

×

343

static int32_t closeStreamIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SInterval* pInterval,

×

346

  qDebug("===stream===close interval window");

×

347

  int32_t                      code = TSDB_CODE_SUCCESS;

×

348

  int32_t                      lino = 0;

×

349

  void*                        pIte = NULL;

×

350

  int32_t                      iter = 0;

×

351

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

352

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

353

  int32_t                      delSize = taosArrayGetSize(pDelWins);

×

354

  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {

×

355

    void*    key = tSimpleHashGetKey(pIte, NULL);

×

356

    SWinKey* pWinKey = (SWinKey*)key;

×

357

    if (delSize > 0) {

×

358

      int32_t index = binarySearchCom(pDelWins, delSize, pWinKey, TSDB_ORDER_DESC, compareWinKey);

×

359

      if (index >= 0 && 0 == compareWinKey(pWinKey, pDelWins, index)) {

×

360

        taosArrayRemove(pDelWins, index);

×

361

        delSize = taosArrayGetSize(pDelWins);

×

365

    void*       chIds = taosHashGet(pPullDataMap, pWinKey, sizeof(SWinKey));

×

366

    STimeWindow win = {

×

367

        .skey = pWinKey->ts,

×

368

        .ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision, NULL) - 1,

×

370

    if (isCloseWindow(&win, pTwSup)) {

×

371

      if (chIds && pPullDataMap) {

×

372

        SArray* chAy = *(SArray**)chIds;

×

373

        int32_t size = taosArrayGetSize(chAy);

×

374

        qDebug("===stream===window %" PRId64 " wait child size:%d", pWinKey->ts, size);

×

375

        for (int32_t i = 0; i < size; i++) {

×

376

          qDebug("===stream===window %" PRId64 " wait child id:%d", pWinKey->ts, *(int32_t*)taosArrayGet(chAy, i));

×

378

        continue;

×

379

      } else if (pPullDataMap) {

×

380

        qDebug("===stream===close window %" PRId64, pWinKey->ts);

×

383

      if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

384

        code = saveWinResult(pWinKey, *(SRowBuffPos**)pIte, closeWins);

×

385

        QUERY_CHECK_CODE(code, lino, _end);

×

387

      int32_t tmpRes = tSimpleHashIterateRemove(pHashMap, pWinKey, sizeof(SWinKey), &pIte, &iter);

×

388

      qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

392

_end:

×

393

  if (code != TSDB_CODE_SUCCESS) {

×

394

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

396

  return code;

×

399

STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) {

×

400

  STimeWindow w = {.skey = ts, .ekey = INT64_MAX};

×

401

  w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision, NULL) - 1;

×

402

  return w;

×

405

static void doBuildDeleteResult(SExecTaskInfo* pTaskInfo, SArray* pWins, int32_t* index,

×

407

  doBuildDeleteResultImpl(&pTaskInfo->storageAPI.stateStore, pTaskInfo->streamInfo.pState, pWins, index, pBlock);

×

408

×

410

void doBuildDeleteResultImpl(SStateStore* pAPI, SStreamState* pState, SArray* pWins, int32_t* index,

×

412

  int32_t code = TSDB_CODE_SUCCESS;

×

413

  int32_t lino = 0;

×

414

  blockDataCleanup(pBlock);

×

415

  int32_t size = taosArrayGetSize(pWins);

×

416

  if (*index == size) {

×

417

    *index = 0;

×

418

    taosArrayClear(pWins);

×

419

    goto _end;

×

421

  code = blockDataEnsureCapacity(pBlock, size - *index);

×

422

  QUERY_CHECK_CODE(code, lino, _end);

×

424

  uint64_t uid = 0;

×

425

  for (int32_t i = *index; i < size; i++) {

×

426

    SWinKey* pWin = taosArrayGet(pWins, i);

×

427

    void*    tbname = NULL;

×

428

    int32_t  winCode = TSDB_CODE_SUCCESS;

×

429

    code = pAPI->streamStateGetParName(pState, pWin->groupId, &tbname, false, &winCode);

×

430

    QUERY_CHECK_CODE(code, lino, _end);

×

432

    if (winCode != TSDB_CODE_SUCCESS) {

×

433

      code = appendDataToSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, NULL);

×

434

      QUERY_CHECK_CODE(code, lino, _end);

×

436

      QUERY_CHECK_CONDITION((tbname), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);

×

438

      STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));

×

439

      code = appendDataToSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, parTbName);

×

440

      QUERY_CHECK_CODE(code, lino, _end);

×

442

    pAPI->streamStateFreeVal(tbname);

×

443

    (*index)++;

×

446

_end:

×

447

  if (code != TSDB_CODE_SUCCESS) {

×

448

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

450

×

452

void destroyFlusedPos(void* pRes) {

×

453

  SRowBuffPos* pPos = (SRowBuffPos*)pRes;

×

454

  if (pPos->needFree && !pPos->pRowBuff) {

×

455

    taosMemoryFreeClear(pPos->pKey);

×

456

    taosMemoryFree(pPos);

×

458

×

460

void destroyFlusedppPos(void* ppRes) {

×

461

  void* pRes = *(void**)ppRes;

×

462

  destroyFlusedPos(pRes);

×

463

×

465

void clearGroupResInfo(SGroupResInfo* pGroupResInfo) {

×

466

  int32_t size = taosArrayGetSize(pGroupResInfo->pRows);

×

467

  if (pGroupResInfo->index >= 0 && pGroupResInfo->index < size) {

×

468

    for (int32_t i = pGroupResInfo->index; i < size; i++) {

×

469

      void* pPos = taosArrayGetP(pGroupResInfo->pRows, i);

×

470

      destroyFlusedPos(pPos);

×

474

  pGroupResInfo->freeItem = false;

×

475

  taosArrayDestroy(pGroupResInfo->pRows);

×

476

  pGroupResInfo->pRows = NULL;

×

477

  pGroupResInfo->index = 0;

×

478

  pGroupResInfo->delIndex = 0;

×

479

×

481

void destroyStreamFinalIntervalOperatorInfo(void* param) {

×

482

  if (param == NULL) {

×

483

    return;

×

485

  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)param;

×

486

  cleanupBasicInfo(&pInfo->binfo);

×

487

  if (pInfo->pOperator) {

×

488

    cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, &pInfo->groupResInfo, &pInfo->aggSup,

×

490

    pInfo->pOperator = NULL;

×

493

  destroyStreamBasicInfo(&pInfo->basic);

×

494

  cleanupAggSup(&pInfo->aggSup);

×

495

  clearGroupResInfo(&pInfo->groupResInfo);

×

496

  taosArrayDestroyP(pInfo->pUpdated, destroyFlusedPos);

×

497

  pInfo->pUpdated = NULL;

×

500

  void* pIte = NULL;

×

501

  while ((pIte = taosHashIterate(pInfo->pPullDataMap, pIte)) != NULL) {

×

502

    taosArrayDestroy(*(void**)pIte);

×

504

  taosHashCleanup(pInfo->pPullDataMap);

×

505

  taosHashCleanup(pInfo->pFinalPullDataMap);

×

506

  taosArrayDestroy(pInfo->pPullWins);

×

507

  blockDataDestroy(pInfo->pPullDataRes);

×

508

  taosArrayDestroy(pInfo->pDelWins);

×

509

  blockDataDestroy(pInfo->pDelRes);

×

510

  blockDataDestroy(pInfo->pMidRetriveRes);

×

511

  blockDataDestroy(pInfo->pMidPulloverRes);

×

512

  if (pInfo->pUpdatedMap != NULL) {

×

514

    tSimpleHashSetFreeFp(pInfo->pUpdatedMap, destroyFlusedppPos);

×

515

    tSimpleHashCleanup(pInfo->pUpdatedMap);

×

516

    pInfo->pUpdatedMap = NULL;

×

519

  if (pInfo->stateStore.streamFileStateDestroy != NULL) {

×

520

    pInfo->stateStore.streamFileStateDestroy(pInfo->pState->pFileState);

×

522

  taosArrayDestroy(pInfo->pMidPullDatas);

×

524

  if (pInfo->pState != NULL && pInfo->pState->dump == 1) {

×

525

    taosMemoryFreeClear(pInfo->pState->pTdbState->pOwner);

×

526

    taosMemoryFreeClear(pInfo->pState->pTdbState);

×

528

  taosMemoryFreeClear(pInfo->pState);

×

530

  nodesDestroyNode((SNode*)pInfo->pPhyNode);

×

531

  colDataDestroy(&pInfo->twAggSup.timeWindowData);

×

532

  cleanupExprSupp(&pInfo->scalarSupp);

×

533

  tSimpleHashCleanup(pInfo->pDeletedMap);

×

535

  blockDataDestroy(pInfo->pCheckpointRes);

×

537

  taosMemoryFreeClear(param);

×

551

void reloadFromDownStream(SOperatorInfo* downstream, SStreamIntervalOperatorInfo* pInfo) {

×

552

  SStateStore* pAPI = &downstream->pTaskInfo->storageAPI.stateStore;

×

554

  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {

×

555

    reloadFromDownStream(downstream->pDownstream[0], pInfo);

×

556

    return;

×

559

  SStreamScanInfo* pScanInfo = downstream->info;

×

560

  pInfo->pUpdateInfo = pScanInfo->pUpdateInfo;

×

563

bool hasSrcPrimaryKeyCol(SSteamOpBasicInfo* pInfo) { return pInfo->primaryPkIndex != -1; }

×

565

int32_t initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamIntervalOperatorInfo* pInfo,

×

567

  int32_t        code = TSDB_CODE_SUCCESS;

×

568

  int32_t        lino = 0;

×

569

  SStateStore*   pAPI = &downstream->pTaskInfo->storageAPI.stateStore;

×

570

  SExecTaskInfo* pTaskInfo = downstream->pTaskInfo;

×

572

  if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) {

×

573

    SStreamPartitionOperatorInfo* pScanInfo = downstream->info;

×

574

    pBasic->primaryPkIndex = pScanInfo->basic.primaryPkIndex;

×

577

  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {

×

578

    return initIntervalDownStream(downstream->pDownstream[0], type, pInfo, pBasic);

×

581

  SStreamScanInfo* pScanInfo = downstream->info;

×

582

  pScanInfo->windowSup.parentType = type;

×

583

  pScanInfo->windowSup.pIntervalAggSup = &pInfo->aggSup;

×

584

  if (!pScanInfo->pUpdateInfo) {

×

585

    code = pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate,

×

586

                                 pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo);

×

587

    QUERY_CHECK_CODE(code, lino, _end);

×

590

  pScanInfo->interval = pInfo->interval;

×

591

  pScanInfo->twAggSup = pInfo->twAggSup;

×

592

  pScanInfo->pState = pInfo->pState;

×

593

  pInfo->pUpdateInfo = pScanInfo->pUpdateInfo;

×

594

  if (!hasSrcPrimaryKeyCol(pBasic)) {

×

595

    pBasic->primaryPkIndex = pScanInfo->basic.primaryPkIndex;

×

598

_end:

×

599

  if (code != TSDB_CODE_SUCCESS) {

×

600

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

602

  return code;

×

605

int32_t compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput,

×

607

  int32_t code = TSDB_CODE_SUCCESS;

×

608

  int32_t lino = 0;

×

609

  for (int32_t k = 0; k < numOfOutput; ++k) {

×

610

    if (fmIsWindowPseudoColumnFunc(pDestCtx[k].functionId)) {

×

611

      if (!pTimeWindowData) {

×

612

        continue;

×

615

      SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pDestCtx[k]);

×

616

      char*                p = GET_ROWCELL_INTERBUF(pEntryInfo);

×

617

      SColumnInfoData      idata = {0};

×

618

      idata.info.type = TSDB_DATA_TYPE_BIGINT;

×

619

      idata.info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes;

×

620

      idata.pData = p;

×

622

      SScalarParam out = {.columnData = &idata};

×

623

      SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData};

×

624

      code = pDestCtx[k].sfp.process(&tw, 1, &out);

×

625

      QUERY_CHECK_CODE(code, lino, _end);

×

627

      pEntryInfo->numOfRes = 1;

×

628

    } else if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) {

×

629

      code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]);

×

630

      QUERY_CHECK_CODE(code, lino, _end);

×

631

    } else if (pDestCtx[k].fpSet.combine == NULL) {

×

632

      char* funName = fmGetFuncName(pDestCtx[k].functionId);

×

633

      qError("%s error, combine funcion for %s is not implemented", GET_TASKID(pTaskInfo), funName);

×

634

      taosMemoryFreeClear(funName);

×

635

      code = TSDB_CODE_FAILED;

×

636

      QUERY_CHECK_CODE(code, lino, _end);

×

640

_end:

×

641

  if (code != TSDB_CODE_SUCCESS) {

×

642

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

644

  return code;

×

647

bool hasIntervalWindow(void* pState, SWinKey* pKey, SStateStore* pStore) {

×

648

  return pStore->streamStateCheck(pState, pKey, false, NULL);

×

651

int32_t setIntervalOutputBuf(void* pState, STimeWindow* win, SRowBuffPos** pResult, int64_t groupId,

×

654

  int32_t code = TSDB_CODE_SUCCESS;

×

655

  int32_t lino = 0;

×

656

  SWinKey key = {.ts = win->skey, .groupId = groupId};

×

657

  char*   value = NULL;

×

658

  int32_t size = pAggSup->resultRowSize;

×

660

  code = pStore->streamStateAddIfNotExist(pState, &key, (void**)&value, &size, pWinCode);

×

661

  QUERY_CHECK_CODE(code, lino, _end);

×

663

  *pResult = (SRowBuffPos*)value;

×

664

  SResultRow* res = (SResultRow*)((*pResult)->pRowBuff);

×

667

  res->win = (*win);

×

668

  code = setResultRowInitCtx(res, pCtx, numOfOutput, rowEntryInfoOffset);

×

669

  QUERY_CHECK_CODE(code, lino, _end);

×

671

_end:

×

672

  if (code != TSDB_CODE_SUCCESS) {

×

673

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

675

  return code;

×

678

bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, STimeWindowAggSupp* pTwSup,

×

680

  if (pTwSup->maxTs != INT64_MIN && pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark) {

×

681

    SWinKey key = {.ts = pWin->skey, .groupId = groupId};

×

682

    if (!hasIntervalWindow(pState, &key, pStore)) {

×

683

      return true;

×

685

    return false;

×

687

  return false;

×

690

int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols, int32_t startPos, TSKEY eKey,

×

693

      getNumOfRowsInTimeWindow(pBlockInfo, tsCols, startPos, eKey, binarySearchForKey, NULL, TSDB_ORDER_ASC);

×

694

  int32_t prevEndPos = forwardRows - 1 + startPos;

×

695

  return getNextQualifiedWindow(pInterval, pNextWin, pBlockInfo, tsCols, prevEndPos, TSDB_ORDER_ASC);

×

698

int32_t addPullWindow(SHashObj* pMap, SWinKey* pWinRes, int32_t size) {

×

699

  int32_t code = TSDB_CODE_SUCCESS;

×

700

  int32_t lino = 0;

×

701

  SArray* childIds = taosArrayInit(8, sizeof(int32_t));

×

702

  QUERY_CHECK_NULL(childIds, code, lino, _end, terrno);

×

703

  for (int32_t i = 0; i < size; i++) {

×

704

    void* tmp = taosArrayPush(childIds, &i);

×

705

    if (!tmp) {

×

706

      code = terrno;

×

707

      QUERY_CHECK_CODE(code, lino, _end);

×

710

  code = taosHashPut(pMap, pWinRes, sizeof(SWinKey), &childIds, sizeof(void*));

×

711

  QUERY_CHECK_CODE(code, lino, _end);

×

712

_end:

×

713

  if (code != TSDB_CODE_SUCCESS) {

×

714

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

716

  return code;

×

719

static void clearStreamIntervalOperator(SStreamIntervalOperatorInfo* pInfo) {

×

720

  tSimpleHashClear(pInfo->aggSup.pResultRowHashTable);

×

721

  clearDiskbasedBuf(pInfo->aggSup.pResultBuf);

×

722

  initResultRowInfo(&pInfo->binfo.resultRowInfo);

×

723

  pInfo->aggSup.currentPageId = -1;

×

724

  pInfo->stateStore.streamStateClear(pInfo->pState);

×

725

×

727

static void clearSpecialDataBlock(SSDataBlock* pBlock) {

×

728

  if (pBlock->info.rows <= 0) {

×

729

    return;

×

731

  blockDataCleanup(pBlock);

×

734

void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pBlock) {

×

735

  int32_t code = TSDB_CODE_SUCCESS;

×

736

  int32_t lino = 0;

×

737

  clearSpecialDataBlock(pBlock);

×

738

  int32_t size = taosArrayGetSize(array);

×

739

  if (size - (*pIndex) == 0) {

×

740

    goto _end;

×

742

  code = blockDataEnsureCapacity(pBlock, size - (*pIndex));

×

743

  QUERY_CHECK_CODE(code, lino, _end);

×

745

  SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);

×

746

  SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);

×

747

  SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);

×

748

  SColumnInfoData* pCalStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);

×

749

  SColumnInfoData* pCalEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);

×

750

  SColumnInfoData* pTbName = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);

×

751

  SColumnInfoData* pPrimaryKey = NULL;

×

752

  if (taosArrayGetSize(pBlock->pDataBlock) > PRIMARY_KEY_COLUMN_INDEX) {

×

753

    pPrimaryKey = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, PRIMARY_KEY_COLUMN_INDEX);

×

755

  for (; (*pIndex) < size; (*pIndex)++) {

×

756

    SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex));

×

757

    code = colDataSetVal(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false);

×

758

    QUERY_CHECK_CODE(code, lino, _end);

×

760

    code = colDataSetVal(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false);

×

761

    QUERY_CHECK_CODE(code, lino, _end);

×

763

    code = colDataSetVal(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false);

×

764

    QUERY_CHECK_CODE(code, lino, _end);

×

766

    code = colDataSetVal(pCalStartTs, pBlock->info.rows, (const char*)&pWin->calWin.skey, false);

×

767

    QUERY_CHECK_CODE(code, lino, _end);

×

769

    code = colDataSetVal(pCalEndTs, pBlock->info.rows, (const char*)&pWin->calWin.ekey, false);

×

770

    QUERY_CHECK_CODE(code, lino, _end);

×

772

    colDataSetNULL(pTbName, pBlock->info.rows);

×

773

    if (pPrimaryKey != NULL) {

×

774

      colDataSetNULL(pPrimaryKey, pBlock->info.rows);

×

777

    pBlock->info.rows++;

×

779

  if ((*pIndex) == size) {

×

780

    *pIndex = 0;

×

781

    taosArrayClear(array);

×

783

  code = blockDataUpdateTsWindow(pBlock, 0);

×

784

  QUERY_CHECK_CODE(code, lino, _end);

×

786

_end:

×

787

  if (code != TSDB_CODE_SUCCESS) {

×

788

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

790

×

792

static int32_t processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval,

×

794

  int32_t                      code = TSDB_CODE_SUCCESS;

×

795

  int32_t                      lino = 0;

×

796

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

797

  SColumnInfoData*             pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);

×

798

  TSKEY*                       tsData = (TSKEY*)pStartCol->pData;

×

799

  SColumnInfoData*             pEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);

×

800

  TSKEY*                       tsEndData = (TSKEY*)pEndCol->pData;

×

801

  SColumnInfoData*             pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);

×

802

  uint64_t*                    groupIdData = (uint64_t*)pGroupCol->pData;

×

803

  int32_t                      chId = getChildIndex(pBlock);

×

804

  bool                         res = false;

×

805

  for (int32_t i = 0; i < pBlock->info.rows; i++) {

×

806

    TSKEY winTs = tsData[i];

×

807

    while (winTs <= tsEndData[i]) {

×

808

      SWinKey winRes = {.ts = winTs, .groupId = groupIdData[i]};

×

809

      void*   chIds = taosHashGet(pMap, &winRes, sizeof(SWinKey));

×

810

      if (chIds) {

×

811

        SArray* chArray = *(SArray**)chIds;

×

812

        int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);

×

813

        if (index != -1) {

×

814

          qDebug("===stream===retrive window %" PRId64 " delete child id %d", winRes.ts, chId);

×

815

          taosArrayRemove(chArray, index);

×

816

          if (taosArrayGetSize(chArray) == 0) {

×

818

            taosArrayDestroy(chArray);

×

819

            int32_t tmpRes = taosHashRemove(pMap, &winRes, sizeof(SWinKey));

×

820

            qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

821

            res = true;

×

822

            qDebug("===stream===retrive pull data over.window %" PRId64, winRes.ts);

×

824

            void* pFinalCh = taosHashGet(pFinalMap, &winRes, sizeof(SWinKey));

×

825

            if (pFinalCh) {

×

826

              int32_t tmpRes = taosHashRemove(pFinalMap, &winRes, sizeof(SWinKey));

×

827

              qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

828

              doDeleteWindow(pOperator, winRes.ts, winRes.groupId);

×

829

              STimeWindow     nextWin = getFinalTimeWindow(winRes.ts, pInterval);

×

830

              SPullWindowInfo pull = {.window = nextWin,

×

831

                                      .groupId = winRes.groupId,

×

832

                                      .calWin.skey = nextWin.skey,

×

833

                                      .calWin.ekey = nextWin.skey};

×

835

              qDebug("===stream===prepare final retrive for delete window:%" PRId64 ",groupId:%" PRId64 ", size:%d",

×

837

              if (IS_MID_INTERVAL_OP(pOperator)) {

×

838

                SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info;

×

840

                void* tmp = taosArrayPush(pInfo->pMidPullDatas, &winRes);

×

841

                if (!tmp) {

×

842

                  code = terrno;

×

843

                  QUERY_CHECK_CODE(code, lino, _end);

×

845

              } else if (savePullWindow(&pull, pPullWins) == TSDB_CODE_SUCCESS) {

×

846

                void* tmp = taosArrayPush(pInfo->pDelWins, &winRes);

×

847

                if (!tmp) {

×

848

                  code = terrno;

×

849

                  QUERY_CHECK_CODE(code, lino, _end);

×

852

                code = addPullWindow(pMap, &winRes, numOfCh);

×

853

                QUERY_CHECK_CODE(code, lino, _end);

×

855

                if (pInfo->destHasPrimaryKey) {

×

856

                  code = tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0);

×

857

                  QUERY_CHECK_CODE(code, lino, _end);

×

859

                qDebug("===stream===prepare final retrive for delete %" PRId64 ", size:%d", winRes.ts, numOfCh);

×

865

      winTs = taosTimeAdd(winTs, pInterval->sliding, pInterval->slidingUnit, pInterval->precision, NULL);

×

868

  if (pBeOver) {

×

869

    *pBeOver = res;

×

872

_end:

×

873

  if (code != TSDB_CODE_SUCCESS) {

×

874

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

876

  return code;

×

879

static int32_t addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, int32_t childId) {

×

880

  int32_t code = TSDB_CODE_SUCCESS;

×

881

  int32_t lino = 0;

×

882

  int32_t size = taosArrayGetSize(wins);

×

883

  for (int32_t i = 0; i < size; i++) {

×

884

    SWinKey*    winKey = taosArrayGet(wins, i);

×

885

    STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval);

×

886

    void*       chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey));

×

887

    if (!chIds) {

×

888

      SPullWindowInfo pull = {

×

889

          .window = nextWin, .groupId = winKey->groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey};

×

891

      if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) {

×

892

        code = addPullWindow(pInfo->pPullDataMap, winKey, pInfo->numOfChild);

×

893

        QUERY_CHECK_CODE(code, lino, _end);

×

895

        if (pInfo->destHasPrimaryKey) {

×

896

          code = tSimpleHashPut(pInfo->pDeletedMap, winKey, sizeof(SWinKey), NULL, 0);

×

897

          QUERY_CHECK_CODE(code, lino, _end);

×

899

        qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, pInfo->numOfChild);

×

902

      SArray* chArray = *(void**)chIds;

×

903

      int32_t index = taosArraySearchIdx(chArray, &childId, compareInt32Val, TD_EQ);

×

904

      qDebug("===stream===check final retrive %" PRId64 ",chid:%d", winKey->ts, index);

×

905

      if (index == -1) {

×

906

        qDebug("===stream===add final retrive %" PRId64, winKey->ts);

×

907

        code = taosHashPut(pInfo->pFinalPullDataMap, winKey, sizeof(SWinKey), NULL, 0);

×

908

        QUERY_CHECK_CODE(code, lino, _end);

×

913

_end:

×

914

  if (code != TSDB_CODE_SUCCESS) {

×

915

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

917

  return code;

×

920

static void clearFunctionContext(SExprSupp* pSup) {

×

921

  for (int32_t i = 0; i < pSup->numOfExprs; i++) {

×

922

    pSup->pCtx[i].saveHandle.currentPage = -1;

×

924

×

926

int32_t getOutputBuf(void* pState, SRowBuffPos* pPos, SResultRow** pResult, SStateStore* pStore) {

×

927

  return pStore->streamStateGetByPos(pState, pPos, (void**)pResult);

×

930

void buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup,

×

932

  int32_t         code = TSDB_CODE_SUCCESS;

×

933

  int32_t         lino = 0;

×

934

  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;

×

935

  SStorageAPI*    pAPI = &pOperator->pTaskInfo->storageAPI;

×

936

  SExprInfo*      pExprInfo = pSup->pExprInfo;

×

937

  int32_t         numOfExprs = pSup->numOfExprs;

×

938

  int32_t*        rowEntryOffset = pSup->rowEntryInfoOffset;

×

939

  SqlFunctionCtx* pCtx = pSup->pCtx;

×

941

  int32_t numOfRows = getNumOfTotalRes(pGroupResInfo);

×

943

  for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) {

×

944

    SRowBuffPos* pPos = *(SRowBuffPos**)taosArrayGet(pGroupResInfo->pRows, i);

×

945

    SResultRow*  pRow = NULL;

×

946

    code = getOutputBuf(pState, pPos, &pRow, &pAPI->stateStore);

×

947

    QUERY_CHECK_CODE(code, lino, _end);

×

948

    uint64_t groupId = ((SWinKey*)pPos->pKey)->groupId;

×

949

    doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset);

×

951

    if (pRow->numOfRows == 0) {

×

952

      pGroupResInfo->index += 1;

×

953

      continue;

×

955

    if (pBlock->info.id.groupId == 0) {

×

956

      pBlock->info.id.groupId = groupId;

×

957

      void*   tbname = NULL;

×

958

      int32_t winCode = TSDB_CODE_SUCCESS;

×

959

      code = pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname,

×

961

      QUERY_CHECK_CODE(code, lino, _end);

×

962

      if (winCode != TSDB_CODE_SUCCESS) {

×

963

        pBlock->info.parTbName[0] = 0;

×

965

        memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);

×

967

      pAPI->stateStore.streamStateFreeVal(tbname);

×

970

      if (pBlock->info.id.groupId != groupId) {

×

971

        break;

×

975

    if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) {

×

976

      break;

×

978

    pGroupResInfo->index += 1;

×

980

    for (int32_t j = 0; j < numOfExprs; ++j) {

×

981

      int32_t slotId = pExprInfo[j].base.resSchema.slotId;

×

983

      pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset);

×

985

      if (pCtx[j].fpSet.finalize) {

×

986

        int32_t tmpRes = pCtx[j].fpSet.finalize(&pCtx[j], pBlock);

×

987

        if (TAOS_FAILED(tmpRes)) {

×

988

          qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(tmpRes));

×

989

          QUERY_CHECK_CODE(code, lino, _end);

×

991

      } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) {

×

996

        SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId);

×

997

        char*            in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo);

×

998

        for (int32_t k = 0; k < pRow->numOfRows; ++k) {

×

999

          code = colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes);

×

1000

          QUERY_CHECK_CODE(code, lino, _end);

×

1005

    if (pSessionKeys) {

×

1006

      SSessionKey key = {.groupId = groupId, .win = pRow->win};

×

1007

      for (int32_t j = 0; j < pRow->numOfRows; ++j) {

×

1008

        const void* px = taosArrayPush(pSessionKeys, &key);

×

1009

        QUERY_CHECK_NULL(px, code, lino, _end, terrno);

×

1013

    pBlock->info.rows += pRow->numOfRows;

×

1016

  pBlock->info.dataLoad = 1;

×

1017

  code = blockDataUpdateTsWindow(pBlock, 0);

×

1018

  QUERY_CHECK_CODE(code, lino, _end);

×

1020

_end:

×

1021

  if (code != TSDB_CODE_SUCCESS) {

×

1022

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

1024

×

1026

void doBuildStreamIntervalResult(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock,

×

1028

  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

×

1030

  pBlock->info.version = pTaskInfo->version;

×

1032

  blockDataCleanup(pBlock);

×

1033

  taosArrayClear(pSessionKeys);

×

1034

  if (!hasRemainResults(pGroupResInfo)) {

×

1035

    return;

×

1039

  pBlock->info.id.groupId = 0;

×

1040

  buildDataBlockFromGroupRes(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo, pSessionKeys);

×

1043

int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,

×

1045

  int32_t startPos = prevPosition + 1;

×

1046

  if (startPos == pDataBlockInfo->rows) {

×

1047

    startPos = -1;

×

1049

    *pNext = getFinalTimeWindow(primaryKeys[startPos], pInterval);

×

1051

  return startPos;

×

1054

static int32_t doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, uint64_t groupId,

×

1056

  int32_t                      code = TSDB_CODE_SUCCESS;

×

1057

  int32_t                      lino = 0;

×

1058

  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info;

×

1059

  pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);

×

1061

  SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo);

×

1062

  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;

×

1063

  SExprSupp*      pSup = &pOperator->exprSupp;

×

1064

  int32_t         numOfOutput = pSup->numOfExprs;

×

1065

  int32_t         step = 1;

×

1066

  TSKEY*          tsCols = NULL;

×

1067

  SRowBuffPos*    pResPos = NULL;

×

1068

  SResultRow*     pResult = NULL;

×

1069

  int32_t         forwardRows = 0;

×

1070

  int32_t         endRowId = pSDataBlock->info.rows - 1;

×

1072

  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);

×

1073

  tsCols = (int64_t*)pColDataInfo->pData;

×

1075

  void*            pPkVal = NULL;

×

1076

  int32_t          pkLen = 0;

×

1077

  SColumnInfoData* pPkColDataInfo = NULL;

×

1078

  if (hasSrcPrimaryKeyCol(&pInfo->basic)) {

×

1079

    pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex);

×

1082

  if (pSDataBlock->info.window.skey != tsCols[0] || pSDataBlock->info.window.ekey != tsCols[endRowId]) {

×

1083

    qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 ",maxKey %" PRId64,

×

1085

    code = blockDataUpdateTsWindow(pSDataBlock, pInfo->primaryTsIndex);

×

1086

    QUERY_CHECK_CODE(code, lino, _end);

×

1089

    if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) {

×

1090

      qError("table uid %" PRIu64 " data block timestamp is out of range! minKey %" PRId64 ",maxKey %" PRId64,

×

1095

  int32_t     startPos = 0;

×

1096

  TSKEY       ts = getStartTsKey(&pSDataBlock->info.window, tsCols);

×

1097

  STimeWindow nextWin = {0};

×

1098

  if (IS_FINAL_INTERVAL_OP(pOperator)) {

×

1099

    nextWin = getFinalTimeWindow(ts, &pInfo->interval);

×

1101

    nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC);

×

1103

  while (1) {

×

1104

    bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);

×

1105

    if (hasSrcPrimaryKeyCol(&pInfo->basic) && !IS_FINAL_INTERVAL_OP(pOperator) && pInfo->ignoreExpiredData &&

×

1106

        pSDataBlock->info.type != STREAM_PULL_DATA) {

×

1107

      pPkVal = colDataGetData(pPkColDataInfo, startPos);

×

1108

      pkLen = colDataGetRowLength(pPkColDataInfo, startPos);

×

1111

    if ((!IS_FINAL_INTERVAL_OP(pOperator) && pInfo->ignoreExpiredData && pSDataBlock->info.type != STREAM_PULL_DATA &&

×

1112

         checkExpiredData(&pInfo->stateStore, pInfo->pUpdateInfo, &pInfo->twAggSup, pSDataBlock->info.id.uid,

×

1113

                          nextWin.ekey, pPkVal, pkLen)) ||

×

1114

        !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) {

×

1115

      startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);

×

1116

      if (startPos < 0) {

×

1117

        break;

×

1119

      qDebug("===stream===ignore expired data, window end ts:%" PRId64 ", maxts - wartermak:%" PRId64, nextWin.ekey,

×

1121

      continue;

×

1124

    if (IS_FINAL_INTERVAL_OP(pOperator) && pInfo->numOfChild > 0) {

×

1125

      bool    ignore = true;

×

1126

      SWinKey winRes = {

×

1127

          .ts = nextWin.skey,

×

1130

      void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey));

×

1131

      if (isDeletedStreamWindow(&nextWin, groupId, pInfo->pState, &pInfo->twAggSup, &pInfo->stateStore) && isClosed &&

×

1132

          !chIds) {

×

1133

        SPullWindowInfo pull = {

×

1134

            .window = nextWin, .groupId = groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey};

×

1136

        if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) {

×

1137

          code = addPullWindow(pInfo->pPullDataMap, &winRes, pInfo->numOfChild);

×

1138

          QUERY_CHECK_CODE(code, lino, _end);

×

1140

          if (pInfo->destHasPrimaryKey) {

×

1141

            code = tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0);

×

1142

            QUERY_CHECK_CODE(code, lino, _end);

×

1146

        int32_t index = -1;

×

1147

        SArray* chArray = NULL;

×

1148

        int32_t chId = 0;

×

1149

        if (chIds) {

×

1150

          chArray = *(void**)chIds;

×

1151

          chId = getChildIndex(pSDataBlock);

×

1152

          index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);

×

1154

        if (index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA) {

×

1155

          ignore = false;

×

1159

      if (ignore) {

×

1160

        startPos = getNextQualifiedFinalWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, startPos);

×

1161

        if (startPos < 0) {

×

1162

          break;

×

1164

        continue;

×

1168

    int32_t winCode = TSDB_CODE_SUCCESS;

×

1169

    code = setIntervalOutputBuf(pInfo->pState, &nextWin, &pResPos, groupId, pSup->pCtx, numOfOutput,

×

1171

    QUERY_CHECK_CODE(code, lino, _end);

×

1173

    if (winCode != TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator) &&

×

1174

        BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {

×

1175

      SSessionKey key = {.win = nextWin, .groupId = groupId};

×

1176

      code = addIntervalAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &key, &pInfo->basic.notifyEventSup,

×

1178

      QUERY_CHECK_CODE(code, lino, _end);

×

1181

    pResult = (SResultRow*)pResPos->pRowBuff;

×

1183

    if (IS_FINAL_INTERVAL_OP(pOperator)) {

×

1184

      forwardRows = 1;

×

1186

      forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey,

×

1190

    SWinKey key = {

×

1191

        .ts = pResult->win.skey,

×

1195

    if (pInfo->destHasPrimaryKey && winCode == TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator)) {

×

1196

      code = tSimpleHashPut(pDeletedMap, &key, sizeof(SWinKey), NULL, 0);

×

1197

      QUERY_CHECK_CODE(code, lino, _end);

×

1200

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) {

×

1201

      code = saveWinResult(&key, pResPos, pUpdatedMap);

×

1202

      QUERY_CHECK_CODE(code, lino, _end);

×

1205

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

1206

      pResPos->beUpdated = true;

×

1207

      code = tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pResPos, POINTER_BYTES);

×

1208

      QUERY_CHECK_CODE(code, lino, _end);

×

1211

    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, 1);

×

1212

    code = applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos,

×

1213

                                           forwardRows, pSDataBlock->info.rows, numOfOutput);

×

1214

    QUERY_CHECK_CODE(code, lino, _end);

×

1215

    key.ts = nextWin.skey;

×

1217

    if (pInfo->delKey.ts > key.ts) {

×

1218

      pInfo->delKey = key;

×

1220

    int32_t prevEndPos = (forwardRows - 1) * step + startPos;

×

1221

    if (IS_FINAL_INTERVAL_OP(pOperator)) {

×

1222

      startPos = getNextQualifiedFinalWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos);

×

1225

          getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC);

×

1227

    if (startPos < 0) {

×

1228

      break;

×

1231

_end:

×

1232

  if (code != TSDB_CODE_SUCCESS) {

×

1233

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

1235

  return code;

×

1238

int winPosCmprImpl(const void* pKey1, const void* pKey2) {

×

1239

  SRowBuffPos* pos1 = *(SRowBuffPos**)pKey1;

×

1240

  SRowBuffPos* pos2 = *(SRowBuffPos**)pKey2;

×

1241

  SWinKey*     pWin1 = (SWinKey*)pos1->pKey;

×

1242

  SWinKey*     pWin2 = (SWinKey*)pos2->pKey;

×

1244

  if (pWin1->groupId > pWin2->groupId) {

×

1245

    return 1;

×

1246

  } else if (pWin1->groupId < pWin2->groupId) {

×

1247

    return -1;

×

1250

  if (pWin1->ts > pWin2->ts) {

×

1251

    return 1;

×

1252

  } else if (pWin1->ts < pWin2->ts) {

×

1253

    return -1;

×

1256

  return 0;

×

1259

static void resetUnCloseWinInfo(SSHashObj* winMap) {

×

1260

  void*   pIte = NULL;

×

1261

  int32_t iter = 0;

×

1262

  while ((pIte = tSimpleHashIterate(winMap, pIte, &iter)) != NULL) {

×

1263

    SRowBuffPos* pPos = *(SRowBuffPos**)pIte;

×

1264

    pPos->beUsed = true;

×

1266

×

1268

int32_t encodeSWinKey(void** buf, SWinKey* key) {

×

1269

  int32_t tlen = 0;

×

1270

  tlen += taosEncodeFixedI64(buf, key->ts);

×

1271

  tlen += taosEncodeFixedU64(buf, key->groupId);

×

1272

  return tlen;

×

1275

void* decodeSWinKey(void* buf, SWinKey* key) {

×

1276

  buf = taosDecodeFixedI64(buf, &key->ts);

×

1277

  buf = taosDecodeFixedU64(buf, &key->groupId);

×

1278

  return buf;

×

1281

int32_t encodeSTimeWindowAggSupp(void** buf, STimeWindowAggSupp* pTwAggSup) {

×

1282

  int32_t tlen = 0;

×

1283

  tlen += taosEncodeFixedI64(buf, pTwAggSup->minTs);

×

1284

  tlen += taosEncodeFixedI64(buf, pTwAggSup->maxTs);

×

1285

  return tlen;

×

1288

void* decodeSTimeWindowAggSupp(void* buf, STimeWindowAggSupp* pTwAggSup) {

×

1289

  buf = taosDecodeFixedI64(buf, &pTwAggSup->minTs);

×

1290

  buf = taosDecodeFixedI64(buf, &pTwAggSup->maxTs);

×

1291

  return buf;

×

1294

int32_t encodeSTimeWindow(void** buf, STimeWindow* pWin) {

×

1295

  int32_t tlen = 0;

×

1296

  tlen += taosEncodeFixedI64(buf, pWin->skey);

×

1297

  tlen += taosEncodeFixedI64(buf, pWin->ekey);

×

1298

  return tlen;

×

1301

void* decodeSTimeWindow(void* buf, STimeWindow* pWin) {

×

1302

  buf = taosDecodeFixedI64(buf, &pWin->skey);

×

1303

  buf = taosDecodeFixedI64(buf, &pWin->ekey);

×

1304

  return buf;

×

1307

int32_t encodeSPullWindowInfo(void** buf, SPullWindowInfo* pPullInfo) {

×

1308

  int32_t tlen = 0;

×

1309

  tlen += encodeSTimeWindow(buf, &pPullInfo->calWin);

×

1310

  tlen += taosEncodeFixedU64(buf, pPullInfo->groupId);

×

1311

  tlen += encodeSTimeWindow(buf, &pPullInfo->window);

×

1312

  return tlen;

×

1315

void* decodeSPullWindowInfo(void* buf, SPullWindowInfo* pPullInfo) {

×

1316

  buf = decodeSTimeWindow(buf, &pPullInfo->calWin);

×

1317

  buf = taosDecodeFixedU64(buf, &pPullInfo->groupId);

×

1318

  buf = decodeSTimeWindow(buf, &pPullInfo->window);

×

1319

  return buf;

×

1322

int32_t encodeSPullWindowInfoArray(void** buf, SArray* pPullInfos) {

×

1323

  int32_t tlen = 0;

×

1324

  int32_t size = taosArrayGetSize(pPullInfos);

×

1325

  tlen += taosEncodeFixedI32(buf, size);

×

1326

  for (int32_t i = 0; i < size; i++) {

×

1327

    void* pItem = taosArrayGet(pPullInfos, i);

×

1328

    tlen += encodeSPullWindowInfo(buf, pItem);

×

1330

  return tlen;

×

1333

int32_t decodeSPullWindowInfoArray(void* buf, SArray* pPullInfos, void** ppBuf) {

×

1334

  int32_t code = TSDB_CODE_SUCCESS;

×

1335

  int32_t lino = 0;

×

1336

  int32_t size = 0;

×

1337

  buf = taosDecodeFixedI32(buf, &size);

×

1338

  for (int32_t i = 0; i < size; i++) {

×

1339

    SPullWindowInfo item = {0};

×

1340

    buf = decodeSPullWindowInfo(buf, &item);

×

1341

    void* tmp = taosArrayPush(pPullInfos, &item);

×

1342

    if (!tmp) {

×

1343

      code = terrno;

×

1344

      QUERY_CHECK_CODE(code, lino, _end);

×

1347

  (*ppBuf) = buf;

×

1349

_end:

×

1350

  if (code != TSDB_CODE_SUCCESS) {

×

1351

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

1353

  return code;

×

1356

int32_t doStreamIntervalEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOperator) {

×

1357

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1358

  if (!pInfo) {

×

1359

    return 0;

×

1362

  void* pData = (buf == NULL) ? NULL : *buf;

×

1365

  int32_t tlen = 0;

×

1366

  int32_t mapSize = tSimpleHashGetSize(pInfo->aggSup.pResultRowHashTable);

×

1367

  tlen += taosEncodeFixedI32(buf, mapSize);

×

1368

  void*   pIte = NULL;

×

1369

  size_t  keyLen = 0;

×

1370

  int32_t iter = 0;

×

1371

  while ((pIte = tSimpleHashIterate(pInfo->aggSup.pResultRowHashTable, pIte, &iter)) != NULL) {

×

1372

    void* key = tSimpleHashGetKey(pIte, &keyLen);

×

1373

    tlen += encodeSWinKey(buf, key);

×

1377

  tlen += encodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);

×

1380

  int32_t size = taosHashGetSize(pInfo->pPullDataMap);

×

1381

  tlen += taosEncodeFixedI32(buf, size);

×

1382

  pIte = NULL;

×

1383

  keyLen = 0;

×

1384

  while ((pIte = taosHashIterate(pInfo->pPullDataMap, pIte)) != NULL) {

×

1385

    void* key = taosHashGetKey(pIte, &keyLen);

×

1386

    tlen += encodeSWinKey(buf, key);

×

1387

    SArray* pArray = *(SArray**)pIte;

×

1388

    int32_t chSize = taosArrayGetSize(pArray);

×

1389

    tlen += taosEncodeFixedI32(buf, chSize);

×

1390

    for (int32_t i = 0; i < chSize; i++) {

×

1391

      void* pChItem = taosArrayGet(pArray, i);

×

1392

      tlen += taosEncodeFixedI32(buf, *(int32_t*)pChItem);

×

1397

  tlen += encodeSPullWindowInfoArray(buf, pInfo->pPullWins);

×

1400

  tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);

×

1403

  tlen += encodeStreamBasicInfo(buf, &pInfo->basic);

×

1406

  if (buf) {

×

1407

    uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));

×

1408

    tlen += taosEncodeFixedU32(buf, cksum);

×

1410

    tlen += sizeof(uint32_t);

×

1413

  return tlen;

×

1416

void doStreamIntervalDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator) {

×

1417

  int32_t                      code = TSDB_CODE_SUCCESS;

×

1418

  int32_t                      lino = 0;

×

1419

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1420

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

1421

  void*                        pDataEnd = POINTER_SHIFT(buf, len);

×

1422

  if (!pInfo) {

×

1423

    code = TSDB_CODE_FAILED;

×

1424

    QUERY_CHECK_CODE(code, lino, _end);

×

1428

  int32_t dataLen = len - sizeof(uint32_t);

×

1429

  void*   pCksum = POINTER_SHIFT(buf, dataLen);

×

1430

  if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) {

×

1431

    code = TSDB_CODE_FAILED;

×

1432

    QUERY_CHECK_CODE(code, lino, _end);

×

1434

  pDataEnd = pCksum;

×

1437

  int32_t mapSize = 0;

×

1438

  buf = taosDecodeFixedI32(buf, &mapSize);

×

1439

  for (int32_t i = 0; i < mapSize; i++) {

×

1440

    SWinKey key = {0};

×

1441

    buf = decodeSWinKey(buf, &key);

×

1442

    SRowBuffPos* pPos = NULL;

×

1443

    int32_t      resSize = pInfo->aggSup.resultRowSize;

×

1444

    int32_t      winCode = TSDB_CODE_SUCCESS;

×

1445

    code = pInfo->stateStore.streamStateAddIfNotExist(pInfo->pState, &key, (void**)&pPos, &resSize, &winCode);

×

1446

    QUERY_CHECK_CODE(code, lino, _end);

×

1447

    QUERY_CHECK_CONDITION((winCode == TSDB_CODE_SUCCESS), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);

×

1449

    code = tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pPos, POINTER_BYTES);

×

1450

    QUERY_CHECK_CODE(code, lino, _end);

×

1454

  buf = decodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);

×

1457

  int32_t size = 0;

×

1458

  buf = taosDecodeFixedI32(buf, &size);

×

1459

  for (int32_t i = 0; i < size; i++) {

×

1460

    SWinKey key = {0};

×

1461

    SArray* pArray = taosArrayInit(0, sizeof(int32_t));

×

1462

    if (!pArray) {

×

1463

      code = terrno;

×

1464

      QUERY_CHECK_CODE(code, lino, _end);

×

1467

    buf = decodeSWinKey(buf, &key);

×

1468

    int32_t chSize = 0;

×

1469

    buf = taosDecodeFixedI32(buf, &chSize);

×

1470

    for (int32_t i = 0; i < chSize; i++) {

×

1471

      int32_t chId = 0;

×

1472

      buf = taosDecodeFixedI32(buf, &chId);

×

1473

      void* tmp = taosArrayPush(pArray, &chId);

×

1474

      if (!tmp) {

×

1475

        code = terrno;

×

1476

        QUERY_CHECK_CODE(code, lino, _end);

×

1479

    code = taosHashPut(pInfo->pPullDataMap, &key, sizeof(SWinKey), &pArray, POINTER_BYTES);

×

1480

    QUERY_CHECK_CODE(code, lino, _end);

×

1484

  code = decodeSPullWindowInfoArray(buf, pInfo->pPullWins, &buf);

×

1485

  QUERY_CHECK_CODE(code, lino, _end);

×

1488

  buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);

×

1491

  if (buf < pDataEnd) {

×

1492

    code = decodeStreamBasicInfo(&buf, &pInfo->basic);

×

1493

    QUERY_CHECK_CODE(code, lino, _end);

×

1496

_end:

×

1497

  if (code != TSDB_CODE_SUCCESS) {

×

1498

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

1500

×

1502

void doStreamIntervalSaveCheckpoint(SOperatorInfo* pOperator) {

×

1503

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1504

  if (needSaveStreamOperatorInfo(&pInfo->basic)) {

×

1505

    int32_t len = doStreamIntervalEncodeOpState(NULL, 0, pOperator);

×

1506

    void*   buf = taosMemoryCalloc(1, len);

×

1507

    if (!buf) {

×

1508

      qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno));

×

1509

      return;

×

1511

    void* pBuf = buf;

×

1512

    len = doStreamIntervalEncodeOpState(&pBuf, len, pOperator);

×

1513

    pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_INTERVAL_OP_CHECKPOINT_NAME,

×

1515

    taosMemoryFree(buf);

×

1516

    saveStreamOperatorStateComplete(&pInfo->basic);

×

1520

int32_t copyIntervalDeleteKey(SSHashObj* pMap, SArray* pWins) {

×

1521

  int32_t code = TSDB_CODE_SUCCESS;

×

1522

  int32_t lino = 0;

×

1523

  void*   pIte = NULL;

×

1524

  int32_t iter = 0;

×

1525

  while ((pIte = tSimpleHashIterate(pMap, pIte, &iter)) != NULL) {

×

1526

    void* pKey = tSimpleHashGetKey(pIte, NULL);

×

1527

    void* tmp = taosArrayPush(pWins, pKey);

×

1528

    if (!tmp) {

×

1529

      code = terrno;

×

1530

      QUERY_CHECK_CODE(code, lino, _end);

×

1533

  tSimpleHashClear(pMap);

×

1535

_end:

×

1536

  if (code != TSDB_CODE_SUCCESS) {

×

1537

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

1539

  return code;

×

1542

static int32_t buildIntervalResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

1543

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1544

  int32_t                      code = TSDB_CODE_SUCCESS;

×

1545

  int32_t                      lino = 0;

×

1546

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

1547

  uint16_t                     opType = pOperator->operatorType;

×

1548

  SStreamNotifyEventSupp*      pNotifySup = &pInfo->basic.notifyEventSup;

×

1549

  STaskNotifyEventStat*        pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;

×

1550

  bool                         addNotifyEvent = false;

×

1553

  if (isTaskKilled(pTaskInfo)) {

×

1554

    (*ppRes) = NULL;

×

1555

    return code;

×

1558

  addNotifyEvent = IS_NORMAL_INTERVAL_OP(pOperator) &&

×

1559

                   BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);

×

1560

  if (IS_FINAL_INTERVAL_OP(pOperator)) {

×

1561

    doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);

×

1562

    if (pInfo->pPullDataRes->info.rows != 0) {

×

1564

      printDataBlock(pInfo->pPullDataRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));

×

1565

      (*ppRes) = pInfo->pPullDataRes;

×

1566

      return code;

×

1570

  doBuildDeleteResult(pTaskInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);

×

1571

  if (pInfo->pDelRes->info.rows != 0) {

×

1573

    printDataBlock(pInfo->pDelRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));

×

1574

    if (addNotifyEvent) {

×

1575

      code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);

×

1576

      QUERY_CHECK_CODE(code, lino, _end);

×

1578

    (*ppRes) = pInfo->pDelRes;

×

1579

    return code;

×

1582

  doBuildStreamIntervalResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo,

×

1584

  if (pInfo->binfo.pRes->info.rows != 0) {

×

1585

    printDataBlock(pInfo->binfo.pRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));

×

1586

    if (addNotifyEvent) {

×

1587

      code = addAggResultNotifyEvent(pInfo->binfo.pRes, pNotifySup->pSessionKeys,

×

1588

                                     pTaskInfo->streamInfo.notifyResultSchema, pNotifySup, pNotifyEventStat);

×

1589

      QUERY_CHECK_CODE(code, lino, _end);

×

1591

    (*ppRes) = pInfo->binfo.pRes;

×

1592

    return code;

×

1595

  code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);

×

1596

  QUERY_CHECK_CODE(code, lino, _end);

×

1597

  if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {

×

1598

    printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

1599

    (*ppRes) = pNotifySup->pEventBlock;

×

1600

    return code;

×

1603

  code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);

×

1604

  QUERY_CHECK_CODE(code, lino, _end);

×

1606

_end:

×

1607

  if (code != TSDB_CODE_SUCCESS) {

×

1608

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

1610

  (*ppRes) = NULL;

×

1611

  return code;

×

1614

int32_t copyUpdateResult(SSHashObj** ppWinUpdated, SArray* pUpdated, __compar_fn_t compar) {

×

1615

  int32_t code = TSDB_CODE_SUCCESS;

×

1616

  int32_t lino = 0;

×

1617

  void*   pIte = NULL;

×

1618

  int32_t iter = 0;

×

1619

  while ((pIte = tSimpleHashIterate(*ppWinUpdated, pIte, &iter)) != NULL) {

×

1620

    void* tmp = taosArrayPush(pUpdated, pIte);

×

1621

    if (!tmp) {

×

1622

      code = terrno;

×

1623

      QUERY_CHECK_CODE(code, lino, _end);

×

1626

  taosArraySort(pUpdated, compar);

×

1627

  tSimpleHashCleanup(*ppWinUpdated);

×

1628

  *ppWinUpdated = NULL;

×

1630

_end:

×

1631

  if (code != TSDB_CODE_SUCCESS) {

×

1632

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

1634

  return code;

×

1637

static int32_t doStreamFinalIntervalAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

1638

  int32_t                      code = TSDB_CODE_SUCCESS;

×

1639

  int32_t                      lino = 0;

×

1640

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1641

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

1642

  SStorageAPI*                 pAPI = &pOperator->pTaskInfo->storageAPI;

×

1644

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

1645

  SExprSupp*     pSup = &pOperator->exprSupp;

×

1647

  qDebug("stask:%s  %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);

×

1649

  if (pOperator->status == OP_EXEC_DONE) {

×

1650

    (*ppRes) = NULL;

×

1651

    return code;

×

1652

  } else if (pOperator->status == OP_RES_TO_RETURN) {

×

1653

    SSDataBlock* resBlock = NULL;

×

1654

    code = buildIntervalResult(pOperator, &resBlock);

×

1655

    QUERY_CHECK_CODE(code, lino, _end);

×

1656

    if (resBlock != NULL) {

×

1657

      (*ppRes) = resBlock;

×

1658

      return code;

×

1661

    if (pInfo->recvGetAll) {

×

1662

      pInfo->recvGetAll = false;

×

1663

      resetUnCloseWinInfo(pInfo->aggSup.pResultRowHashTable);

×

1666

    if (pInfo->reCkBlock) {

×

1667

      pInfo->reCkBlock = false;

×

1668

      printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

1669

      (*ppRes) = pInfo->pCheckpointRes;

×

1670

      return code;

×

1673

    setStreamOperatorCompleted(pOperator);

×

1674

    if (!IS_FINAL_INTERVAL_OP(pOperator)) {

×

1675

      clearFunctionContext(&pOperator->exprSupp);

×

1677

      clearStreamIntervalOperator(pInfo);

×

1678

      qDebug("===stream===clear semi operator");

×

1680

    (*ppRes) = NULL;

×

1681

    return code;

×

1683

    if (!IS_FINAL_INTERVAL_OP(pOperator)) {

×

1684

      SSDataBlock* resBlock = NULL;

×

1685

      code = buildIntervalResult(pOperator, &resBlock);

×

1686

      QUERY_CHECK_CODE(code, lino, _end);

×

1687

      if (resBlock != NULL) {

×

1688

        (*ppRes) = resBlock;

×

1689

        return code;

×

1692

      if (pInfo->recvRetrive) {

×

1693

        pInfo->recvRetrive = false;

×

1694

        printDataBlock(pInfo->pMidRetriveRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

1695

        (*ppRes) = pInfo->pMidRetriveRes;

×

1696

        return code;

×

1701

  if (!pInfo->pUpdated) {

×

1702

    pInfo->pUpdated = taosArrayInit(4096, POINTER_BYTES);

×

1703

    QUERY_CHECK_NULL(pInfo->pUpdated, code, lino, _end, terrno);

×

1705

  if (!pInfo->pUpdatedMap) {

×

1706

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

1707

    pInfo->pUpdatedMap = tSimpleHashInit(4096, hashFn);

×

1708

    QUERY_CHECK_NULL(pInfo->pUpdatedMap, code, lino, _end, terrno);

×

1711

  while (1) {

×

1712

    if (isTaskKilled(pTaskInfo)) {

×

1713

      qInfo("===stream=== %s task is killed, code %s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code));

×

1714

      (*ppRes) = NULL;

×

1715

      return code;

×

1718

    SSDataBlock* pBlock = NULL;

×

1719

    code = downstream->fpSet.getNextFn(downstream, &pBlock);

×

1720

    QUERY_CHECK_CODE(code, lino, _end);

×

1722

    if (pBlock == NULL) {

×

1723

      pOperator->status = OP_RES_TO_RETURN;

×

1724

      qDebug("===stream===return data:%s. recv datablock num:%" PRIu64, getStreamOpName(pOperator->operatorType),

×

1726

      pInfo->numOfDatapack = 0;

×

1727

      break;

×

1730

    pInfo->numOfDatapack++;

×

1731

    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));

×

1732

    setStreamOperatorState(&pInfo->basic, pBlock->info.type);

×

1734

    if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) {

×

1735

      pInfo->binfo.pRes->info.type = pBlock->info.type;

×

1736

    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||

×

1737

               pBlock->info.type == STREAM_CLEAR) {

×

1738

      SArray* delWins = taosArrayInit(8, sizeof(SWinKey));

×

1739

      QUERY_CHECK_NULL(delWins, code, lino, _end, terrno);

×

1740

      SHashObj* finalMap = IS_FINAL_INTERVAL_OP(pOperator) ? pInfo->pFinalPullDataMap : NULL;

×

1741

      code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, finalMap);

×

1742

      QUERY_CHECK_CODE(code, lino, _end);

×

1744

      if (IS_FINAL_INTERVAL_OP(pOperator)) {

×

1745

        int32_t chId = getChildIndex(pBlock);

×

1746

        code = addRetriveWindow(delWins, pInfo, chId);

×

1747

        QUERY_CHECK_CODE(code, lino, _end);

×

1749

        if (pBlock->info.type != STREAM_CLEAR) {

×

1750

          void* tmp = taosArrayAddAll(pInfo->pDelWins, delWins);

×

1751

          if (!tmp && taosArrayGetSize(delWins) > 0) {

×

1752

            code = TSDB_CODE_OUT_OF_MEMORY;

×

1753

            QUERY_CHECK_CODE(code, lino, _end);

×

1756

        taosArrayDestroy(delWins);

×

1757

        continue;

×

1759

      removeResults(delWins, pInfo->pUpdatedMap);

×

1760

      void* tmp = taosArrayAddAll(pInfo->pDelWins, delWins);

×

1761

      if (!tmp && taosArrayGetSize(delWins) > 0) {

×

1762

        code = TSDB_CODE_OUT_OF_MEMORY;

×

1763

        QUERY_CHECK_CODE(code, lino, _end);

×

1765

      taosArrayDestroy(delWins);

×

1767

      doBuildDeleteResult(pTaskInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);

×

1768

      if (pInfo->pDelRes->info.rows != 0) {

×

1770

        printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

1771

        if (pBlock->info.type == STREAM_CLEAR) {

×

1772

          pInfo->pDelRes->info.type = STREAM_CLEAR;

×

1774

          pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;

×

1776

        (*ppRes) = pInfo->pDelRes;

×

1777

        return code;

×

1780

      break;

×

1781

    } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_INTERVAL_OP(pOperator)) {

×

1782

      pInfo->recvGetAll = true;

×

1783

      code = getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap);

×

1784

      QUERY_CHECK_CODE(code, lino, _end);

×

1785

      continue;

×

1786

    } else if (pBlock->info.type == STREAM_RETRIEVE) {

×

1787

      if (!IS_FINAL_INTERVAL_OP(pOperator)) {

×

1788

        pInfo->recvRetrive = true;

×

1789

        code = copyDataBlock(pInfo->pMidRetriveRes, pBlock);

×

1790

        QUERY_CHECK_CODE(code, lino, _end);

×

1792

        pInfo->pMidRetriveRes->info.type = STREAM_MID_RETRIEVE;

×

1793

        code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, NULL, pInfo->pUpdatedMap, NULL);

×

1794

        QUERY_CHECK_CODE(code, lino, _end);

×

1795

        break;

×

1797

      continue;

×

1798

    } else if (pBlock->info.type == STREAM_PULL_OVER && IS_FINAL_INTERVAL_OP(pOperator)) {

×

1799

      code = processPullOver(pBlock, pInfo->pPullDataMap, pInfo->pFinalPullDataMap, &pInfo->interval, pInfo->pPullWins,

×

1801

      QUERY_CHECK_CODE(code, lino, _end);

×

1802

      continue;

×

1803

    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {

×

1804

      (*ppRes) = pBlock;

×

1805

      return code;

×

1806

    } else if (pBlock->info.type == STREAM_CHECKPOINT) {

×

1807

      pAPI->stateStore.streamStateCommit(pInfo->pState);

×

1808

      doStreamIntervalSaveCheckpoint(pOperator);

×

1809

      code = copyDataBlock(pInfo->pCheckpointRes, pBlock);

×

1810

      QUERY_CHECK_CODE(code, lino, _end);

×

1812

      continue;

×

1813

    } else if (IS_FINAL_INTERVAL_OP(pOperator) && pBlock->info.type == STREAM_MID_RETRIEVE) {

×

1814

      continue;

×

1816

      if (pBlock->info.type != STREAM_INVALID) {

×

1817

        code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;

×

1818

        QUERY_CHECK_CODE(code, lino, _end);

×

1822

    if (pInfo->scalarSupp.pExprInfo != NULL) {

×

1823

      SExprSupp* pExprSup = &pInfo->scalarSupp;

×

1824

      code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);

×

1825

      QUERY_CHECK_CODE(code, lino, _end);

×

1827

    code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

1828

    QUERY_CHECK_CODE(code, lino, _end);

×

1830

    code = doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pInfo->pUpdatedMap, pInfo->pDeletedMap);

×

1831

    if (code == TSDB_CODE_STREAM_INTERNAL_ERROR) {

×

1832

      code = TSDB_CODE_SUCCESS;

×

1833

      pOperator->status = OP_RES_TO_RETURN;

×

1834

      break;

×

1836

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);

×

1837

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.watermark);

×

1838

    pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey);

×

1841

  if (IS_FINAL_INTERVAL_OP(pOperator) && !pInfo->destHasPrimaryKey) {

×

1842

    removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins);

×

1844

  if (IS_FINAL_INTERVAL_OP(pOperator)) {

×

1845

    code = closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval,

×

1847

    QUERY_CHECK_CODE(code, lino, _end);

×

1849

    if (pInfo->destHasPrimaryKey) {

×

1850

      code = copyIntervalDeleteKey(pInfo->pDeletedMap, pInfo->pDelWins);

×

1851

      QUERY_CHECK_CODE(code, lino, _end);

×

1854

  pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs;

×

1856

  code = copyUpdateResult(&pInfo->pUpdatedMap, pInfo->pUpdated, winPosCmprImpl);

×

1857

  QUERY_CHECK_CODE(code, lino, _end);

×

1859

  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);

×

1860

  pInfo->pUpdated = NULL;

×

1861

  code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);

×

1862

  QUERY_CHECK_CODE(code, lino, _end);

×

1864

  SSDataBlock* resBlock = NULL;

×

1865

  code = buildIntervalResult(pOperator, &resBlock);

×

1866

  QUERY_CHECK_CODE(code, lino, _end);

×

1867

  if (resBlock != NULL) {

×

1868

    (*ppRes) = resBlock;

×

1869

    return code;

×

1872

  if (pInfo->recvRetrive) {

×

1873

    pInfo->recvRetrive = false;

×

1874

    printDataBlock(pInfo->pMidRetriveRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

1875

    (*ppRes) = pInfo->pMidRetriveRes;

×

1876

    return code;

×

1879

_end:

×

1880

  if (code != TSDB_CODE_SUCCESS) {

×

1881

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

1882

    pTaskInfo->code = code;

×

1883

    T_LONG_JMP(pTaskInfo->env, code);

×

1885

  setStreamOperatorCompleted(pOperator);

×

1886

  (*ppRes) = NULL;

×

1887

  return code;

×

1890

int64_t getDeleteMark(SWindowPhysiNode* pWinPhyNode, int64_t interval) {

×

1891

  if (pWinPhyNode->deleteMark <= 0) {

×

1892

    return DEAULT_DELETE_MARK;

×

1894

  int64_t deleteMark = TMAX(pWinPhyNode->deleteMark, pWinPhyNode->watermark);

×

1895

  deleteMark = TMAX(deleteMark, interval);

×

1896

  return deleteMark;

×

1899

int64_t getDeleteMarkFromOption(SStreamNodeOption* pOption) {

×

1900

  if (pOption->deleteMark <= 0) {

×

1901

    return DEAULT_DELETE_MARK;

×

1903

  int64_t deleteMark = TMAX(pOption->deleteMark, pOption->watermark);

×

1904

  return deleteMark;

×

1907

TSKEY compareTs(void* pKey) {

×

1908

  SWinKey* pWinKey = (SWinKey*)pKey;

×

1909

  return pWinKey->ts;

×

1912

static int32_t getSelectivityBufSize(SqlFunctionCtx* pCtx) {

×

1913

  if (pCtx->subsidiaries.rowLen == 0) {

×

1914

    int32_t rowLen = 0;

×

1915

    for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) {

×

1916

      SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j];

×

1917

      rowLen += pc->pExpr->base.resSchema.bytes;

×

1920

    return rowLen + pCtx->subsidiaries.num * sizeof(bool);

×

1922

    return pCtx->subsidiaries.rowLen;

×

1926

static int32_t getMaxFunResSize(SExprSupp* pSup, int32_t numOfCols) {

×

1927

  int32_t size = 0;

×

1928

  for (int32_t i = 0; i < numOfCols; ++i) {

×

1929

    int32_t resSize = getSelectivityBufSize(pSup->pCtx + i);

×

1930

    size = TMAX(size, resSize);

×

1932

  return size;

×

1935

static void streamIntervalReleaseState(SOperatorInfo* pOperator) {

×

1936

  if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL &&

×

1937

      pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {

×

1938

    SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1939

    int32_t                      resSize = sizeof(TSKEY);

×

1940

    pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME,

×

1941

                                          strlen(STREAM_INTERVAL_OP_STATE_NAME), &pInfo->twAggSup.maxTs, resSize);

×

1943

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1944

  SStorageAPI*                 pAPI = &pOperator->pTaskInfo->storageAPI;

×

1945

  pAPI->stateStore.streamStateCommit(pInfo->pState);

×

1946

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

1947

  if (downstream->fpSet.releaseStreamStateFn) {

×

1948

    downstream->fpSet.releaseStreamStateFn(downstream);

×

1950

×

1952

void streamIntervalReloadState(SOperatorInfo* pOperator) {

×

1953

  int32_t                      code = TSDB_CODE_SUCCESS;

×

1954

  int32_t                      lino = 0;

×

1955

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1956

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

1957

  if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL &&

×

1958

      pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {

×

1959

    int32_t size = 0;

×

1960

    void*   pBuf = NULL;

×

1961

    code = pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME,

×

1963

    QUERY_CHECK_CODE(code, lino, _end);

×

1965

    TSKEY ts = *(TSKEY*)pBuf;

×

1966

    taosMemoryFree(pBuf);

×

1967

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);

×

1968

    pInfo->stateStore.streamStateReloadInfo(pInfo->pState, ts);

×

1970

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

1971

  if (downstream->fpSet.reloadStreamStateFn) {

×

1972

    downstream->fpSet.reloadStreamStateFn(downstream);

×

1974

  reloadFromDownStream(downstream, pInfo);

×

1976

_end:

×

1977

  if (code != TSDB_CODE_SUCCESS) {

×

1978

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

1980

×

1982

int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,

×

1984

  QRY_PARAM_CHECK(pOptrInfo);

×

1986

  int32_t                      code = TSDB_CODE_SUCCESS;

×

1987

  int32_t                      lino = 0;

×

1988

  SIntervalPhysiNode*          pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;

×

1989

  SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo));

×

1990

  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));

×

1991

  if (pInfo == NULL || pOperator == NULL) {

×

1992

    code = terrno;

×

1993

    QUERY_CHECK_CODE(code, lino, _error);

×

1996

  pOperator->exprSupp.hasWindowOrGroup = true;

×

1997

  pOperator->pTaskInfo = pTaskInfo;

×

1998

  SStorageAPI* pAPI = &pTaskInfo->storageAPI;

×

2000

  pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval,

×

2001

                                .sliding = pIntervalPhyNode->sliding,

×

2002

                                .intervalUnit = pIntervalPhyNode->intervalUnit,

×

2003

                                .slidingUnit = pIntervalPhyNode->slidingUnit,

×

2004

                                .offset = pIntervalPhyNode->offset,

×

2005

                                .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision,

×

2006

                                .timeRange = pIntervalPhyNode->timeRange};

×

2007

  calcIntervalAutoOffset(&pInfo->interval);

×

2008

  pInfo->twAggSup = (STimeWindowAggSupp){

×

2009

      .waterMark = pIntervalPhyNode->window.watermark,

×

2010

      .calTrigger = pIntervalPhyNode->window.triggerType,

×

2013

      .deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval),

×

2017

  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;

×

2018

  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;

×

2019

  initResultSizeInfo(&pOperator->resultInfo, 4096);

×

2020

  if (pIntervalPhyNode->window.pExprs != NULL) {

×

2021

    int32_t    numOfScalar = 0;

×

2022

    SExprInfo* pScalarExprInfo = NULL;

×

2024

    code = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalar);

×

2025

    QUERY_CHECK_CODE(code, lino, _error);

×

2027

    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);

×

2028

    QUERY_CHECK_CODE(code, lino, _error);

×

2031

  SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);

×

2032

  QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);

×

2033

  initBasicInfo(&pInfo->binfo, pResBlock);

×

2035

  pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState));

×

2036

  QUERY_CHECK_NULL(pInfo->pState, code, lino, _error, terrno);

×

2037

  qInfo("open state %p", pInfo->pState);

×

2038

  pAPI->stateStore.streamStateCopyBackend(pTaskInfo->streamInfo.pState, pInfo->pState);

×

2041

  qInfo("copy state %p to %p", pTaskInfo->streamInfo.pState, pInfo->pState);

×

2043

  pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsIndex);

×

2045

  int32_t    numOfCols = 0;

×

2046

  SExprInfo* pExprInfo = NULL;

×

2047

  code = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);

×

2048

  QUERY_CHECK_CODE(code, lino, _error);

×

2050

  code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,

×

2051

                    pInfo->pState, &pTaskInfo->storageAPI.functionStore);

×

2052

  QUERY_CHECK_CODE(code, lino, _error);

×

2054

  code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

×

2055

  QUERY_CHECK_CODE(code, lino, _error);

×

2056

  initResultRowInfo(&pInfo->binfo.resultRowInfo);

×

2058

  pInfo->numOfChild = numOfChild;

×

2059

  pInfo->pPhyNode = NULL;

×

2060

  code = nodesCloneNode((SNode*)pPhyNode, (SNode**)&pInfo->pPhyNode);

×

2061

  if (TSDB_CODE_SUCCESS != code) {

×

2062

    goto _error;

×

2065

  pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo));

×

2066

  QUERY_CHECK_NULL(pInfo->pPullWins, code, lino, _error, terrno);

×

2067

  pInfo->pullIndex = 0;

×

2068

  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

2069

  pInfo->pPullDataMap = taosHashInit(64, hashFn, true, HASH_NO_LOCK);

×

2070

  pInfo->pFinalPullDataMap = taosHashInit(64, hashFn, true, HASH_NO_LOCK);

×

2072

  code = createSpecialDataBlock(STREAM_RETRIEVE, &pInfo->pPullDataRes);

×

2073

  QUERY_CHECK_CODE(code, lino, _error);

×

2075

  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;

×

2076

  pInfo->ignoreExpiredDataSaved = false;

×

2077

  code = createSpecialDataBlock(STREAM_DELETE_RESULT, &pInfo->pDelRes);

×

2078

  QUERY_CHECK_CODE(code, lino, _error);

×

2080

  pInfo->delIndex = 0;

×

2081

  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));

×

2082

  QUERY_CHECK_NULL(pInfo->pDelWins, code, lino, _error, terrno);

×

2083

  pInfo->delKey.ts = INT64_MAX;

×

2084

  pInfo->delKey.groupId = 0;

×

2085

  pInfo->numOfDatapack = 0;

×

2086

  pInfo->pUpdated = NULL;

×

2087

  pInfo->pUpdatedMap = NULL;

×

2088

  pInfo->stateStore = pTaskInfo->storageAPI.stateStore;

×

2089

  int32_t funResSize = getMaxFunResSize(&pOperator->exprSupp, numOfCols);

×

2090

  pInfo->pState->pFileState = NULL;

×

2093

  pInfo->pState->pResultRowStore.resultRowGet = getResultRowFromBuf;

×

2094

  pInfo->pState->pResultRowStore.resultRowPut = putResultRowToBuf;

×

2095

  pInfo->pState->pExprSupp = &pOperator->exprSupp;

×

2098

      pAPI->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize,

×

2099

                                           compareTs, pInfo->pState, pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo),

×

2100

                                           pHandle->checkpointId, STREAM_STATE_BUFF_HASH, &pInfo->pState->pFileState);

×

2101

  QUERY_CHECK_CODE(code, lino, _error);

×

2103

  pInfo->dataVersion = 0;

×

2104

  pInfo->recvGetAll = false;

×

2105

  pInfo->recvPullover = false;

×

2106

  pInfo->recvRetrive = false;

×

2108

  code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);

×

2109

  QUERY_CHECK_CODE(code, lino, _error);

×

2110

  code = createSpecialDataBlock(STREAM_MID_RETRIEVE, &pInfo->pMidRetriveRes);

×

2111

  QUERY_CHECK_CODE(code, lino, _error);

×

2112

  code = createSpecialDataBlock(STREAM_MID_RETRIEVE, &pInfo->pMidPulloverRes);

×

2113

  QUERY_CHECK_CODE(code, lino, _error);

×

2115

  pInfo->clearState = false;

×

2116

  pInfo->pMidPullDatas = taosArrayInit(4, sizeof(SWinKey));

×

2117

  QUERY_CHECK_NULL(pInfo->pMidPullDatas, code, lino, _error, terrno);

×

2118

  pInfo->pDeletedMap = tSimpleHashInit(4096, hashFn);

×

2119

  QUERY_CHECK_NULL(pInfo->pDeletedMap, code, lino, _error, terrno);

×

2120

  pInfo->destHasPrimaryKey = pIntervalPhyNode->window.destHasPrimaryKey;

×

2121

  pInfo->pOperator = pOperator;

×

2123

  pOperator->operatorType = pPhyNode->type;

×

2124

  if (!IS_FINAL_INTERVAL_OP(pOperator) || numOfChild == 0) {

×

2125

    pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;

×

2127

  pOperator->name = getStreamOpName(pOperator->operatorType);

×

2128

  pOperator->blocking = true;

×

2129

  pOperator->status = OP_NOT_OPENED;

×

2130

  pOperator->info = pInfo;

×

2132

  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {

×

2134

        createOperatorFpSet(NULL, doStreamMidIntervalAggNext, NULL, destroyStreamFinalIntervalOperatorInfo,

×

2138

        createOperatorFpSet(NULL, doStreamFinalIntervalAggNext, NULL, destroyStreamFinalIntervalOperatorInfo,

×

2141

  setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState);

×

2143

  code = initStreamBasicInfo(&pInfo->basic, pOperator);

×

2144

  QUERY_CHECK_CODE(code, lino, _error);

×

2146

  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL ||

×

2147

      pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {

×

2148

    pInfo->basic.primaryPkIndex = -1;

×

2149

    code = initIntervalDownStream(downstream, pPhyNode->type, pInfo, &pInfo->basic);

×

2150

    QUERY_CHECK_CODE(code, lino, _error);

×

2152

  code = appendDownstream(pOperator, &downstream, 1);

×

2153

  QUERY_CHECK_CODE(code, lino, _error);

×

2156

  void*   buff = NULL;

×

2157

  int32_t len = 0;

×

2158

  int32_t res = pAPI->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_CHECKPOINT_NAME,

×

2160

  if (res == TSDB_CODE_SUCCESS) {

×

2161

    doStreamIntervalDecodeOpState(buff, len, pOperator);

×

2162

    taosMemoryFree(buff);

×

2165

  *pOptrInfo = pOperator;

×

2166

  return TSDB_CODE_SUCCESS;

×

2168

_error:

×

2169

  if (pInfo != NULL) destroyStreamFinalIntervalOperatorInfo(pInfo);

×

2170

  destroyOperatorAndDownstreams(pOperator, &downstream, 1);

×

2171

  pTaskInfo->code = code;

×

2172

  return code;

×

2175

void destroyStreamAggSupporter(SStreamAggSupporter* pSup) {

×

2176

  tSimpleHashCleanup(pSup->pResultRows);

×

2177

  destroyDiskbasedBuf(pSup->pResultBuf);

×

2178

  blockDataDestroy(pSup->pScanBlock);

×

2179

  if (pSup->stateStore.streamFileStateDestroy != NULL) {

×

2180

    pSup->stateStore.streamFileStateDestroy(pSup->pState->pFileState);

×

2182

  taosMemoryFreeClear(pSup->pState);

×

2183

  taosMemoryFreeClear(pSup->pDummyCtx);

×

2184

×

2186

void destroyResultWinInfo(void* pRes) {

×

2187

  SResultWindowInfo* pWinRes = (SResultWindowInfo*)pRes;

×

2188

  destroyFlusedPos(pWinRes->pStatePos);

×

2189

×

2191

void clearSessionGroupResInfo(SGroupResInfo* pGroupResInfo) {

×

2192

  int32_t size = taosArrayGetSize(pGroupResInfo->pRows);

×

2193

  if (pGroupResInfo->index >= 0 && pGroupResInfo->index < size) {

×

2194

    for (int32_t i = pGroupResInfo->index; i < size; i++) {

×

2195

      SResultWindowInfo* pRes = (SResultWindowInfo*) taosArrayGet(pGroupResInfo->pRows, i);

×

2196

      destroyFlusedPos(pRes->pStatePos);

×

2197

      pRes->pStatePos = NULL;

×

2201

  pGroupResInfo->freeItem = false;

×

2202

  taosArrayDestroy(pGroupResInfo->pRows);

×

2203

  pGroupResInfo->pRows = NULL;

×

2204

  pGroupResInfo->index = 0;

×

2205

×

2207

void destroyStreamSessionAggOperatorInfo(void* param) {

×

2208

  if (param == NULL) {

×

2209

    return;

×

2211

  SStreamSessionAggOperatorInfo* pInfo = (SStreamSessionAggOperatorInfo*)param;

×

2212

  cleanupBasicInfo(&pInfo->binfo);

×

2213

  if (pInfo->pOperator) {

×

2214

    cleanupResultInfoInStream(pInfo->pOperator->pTaskInfo, pInfo->streamAggSup.pState, &pInfo->pOperator->exprSupp,

×

2216

    pInfo->pOperator = NULL;

×

2219

  destroyStreamBasicInfo(&pInfo->basic);

×

2220

  cleanupExprSupp(&pInfo->scalarSupp);

×

2221

  clearSessionGroupResInfo(&pInfo->groupResInfo);

×

2222

  taosArrayDestroyEx(pInfo->pUpdated, destroyResultWinInfo);

×

2223

  pInfo->pUpdated = NULL;

×

2224

  destroyStreamAggSupporter(&pInfo->streamAggSup);

×

2226

  if (pInfo->pChildren != NULL) {

×

2227

    int32_t size = taosArrayGetSize(pInfo->pChildren);

×

2228

    for (int32_t i = 0; i < size; i++) {

×

2229

      SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, i);

×

2230

      destroyOperator(pChild);

×

2232

    taosArrayDestroy(pInfo->pChildren);

×

2235

  colDataDestroy(&pInfo->twAggSup.timeWindowData);

×

2236

  blockDataDestroy(pInfo->pDelRes);

×

2237

  blockDataDestroy(pInfo->pWinBlock);

×

2238

  tSimpleHashCleanup(pInfo->pStUpdated);

×

2239

  tSimpleHashCleanup(pInfo->pStDeleted);

×

2240

  cleanupGroupResInfo(&pInfo->groupResInfo);

×

2242

  taosArrayDestroy(pInfo->historyWins);

×

2243

  blockDataDestroy(pInfo->pCheckpointRes);

×

2244

  tSimpleHashCleanup(pInfo->pPkDeleted);

×

2245

  destroyNonBlockAggSupptor(&pInfo->nbSup);

×

2247

  taosMemoryFreeClear(param);

×

2250

int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfCols,

×

2252

  initBasicInfo(pBasicInfo, pResultBlock);

×

2253

  int32_t code = initExprSupp(pSup, pExprInfo, numOfCols, pStore);

×

2254

  if (code != TSDB_CODE_SUCCESS) {

×

2255

    return code;

×

2258

  for (int32_t i = 0; i < numOfCols; ++i) {

×

2259

    pSup->pCtx[i].saveHandle.pBuf = NULL;

×

2262

  return TSDB_CODE_SUCCESS;

×

2265

void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t nums) {

×

2266

  for (int i = 0; i < nums; i++) {

×

2267

    pDummy[i].functionId = pCtx[i].functionId;

×

2268

    pDummy[i].isNotNullFunc = pCtx[i].isNotNullFunc;

×

2269

    pDummy[i].isPseudoFunc = pCtx[i].isPseudoFunc;

×

2270

    pDummy[i].fpSet.init = pCtx[i].fpSet.init;

×

2272

×

2274

int32_t initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex,

×

2276

  SExecTaskInfo* pTaskInfo = downstream->pTaskInfo;

×

2277

  int32_t        code = TSDB_CODE_SUCCESS;

×

2278

  int32_t        lino = 0;

×

2279

  if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) {

×

2280

    SStreamPartitionOperatorInfo* pScanInfo = downstream->info;

×

2281

    pScanInfo->tsColIndex = tsColIndex;

×

2282

    pBasic->primaryPkIndex = pScanInfo->basic.primaryPkIndex;

×

2285

  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {

×

2286

    code = initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup, pBasic, recalculateInterval);

×

2287

    return code;

×

2289

  SStreamScanInfo* pScanInfo = downstream->info;

×

2290

  pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type};

×

2291

  pScanInfo->pState = pAggSup->pState;

×

2292

  if (!pScanInfo->pUpdateInfo) {

×

2293

    code = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark,

×

2294

                                              pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen,

×

2295

                                              &pScanInfo->pUpdateInfo);

×

2296

    QUERY_CHECK_CODE(code, lino, _end);

×

2298

  pScanInfo->twAggSup = *pTwSup;

×

2299

  pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo;

×

2300

  if (!hasSrcPrimaryKeyCol(pBasic)) {

×

2301

    pBasic->primaryPkIndex = pScanInfo->basic.primaryPkIndex;

×

2304

  pBasic->pTsDataState = pScanInfo->basic.pTsDataState;

×

2306

  if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_SESSION) {

×

2307

    pScanInfo->scanAllTables = true;

×

2309

  pScanInfo->recalculateInterval = recalculateInterval;

×

2310

  pScanInfo->windowSup.parentType = type;

×

2311

  pScanInfo->recParam.gap = pAggSup->gap;

×

2313

_end:

×

2314

  if (code != TSDB_CODE_SUCCESS) {

×

2315

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

2317

  return code;

×

2320

static TSKEY sesionTs(void* pKey) {

×

2321

  SSessionKey* pWinKey = (SSessionKey*)pKey;

×

2322

  return pWinKey->win.skey;

×

2325

int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, int32_t numOfOutput, int64_t gap,

×

2329

  pSup->resultRowSize = (keySize + getResultRowSize(pExpSup->pCtx, numOfOutput)) * ratio;

×

2330

  int32_t lino = 0;

×

2331

  int32_t code = createSpecialDataBlock(STREAM_CLEAR, &pSup->pScanBlock);

×

2332

  QUERY_CHECK_CODE(code, lino, _end);

×

2334

  pSup->gap = gap;

×

2335

  pSup->stateKeySize = keySize;

×

2336

  pSup->stateKeyType = keyType;

×

2337

  pSup->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfOutput, sizeof(SqlFunctionCtx));

×

2338

  QUERY_CHECK_NULL(pSup->pDummyCtx, code, lino, _end, terrno);

×

2340

  pSup->stateStore = *pStore;

×

2341

  pSup->pSessionAPI = pApi;

×

2343

  initDummyFunction(pSup->pDummyCtx, pExpSup->pCtx, numOfOutput);

×

2344

  pSup->pState = taosMemoryCalloc(1, sizeof(SStreamState));

×

2345

  QUERY_CHECK_NULL(pSup->pState, code, lino, _end, terrno);

×

2347

  *(pSup->pState) = *pState;

×

2348

  pSup->stateStore.streamStateSetNumber(pSup->pState, -1, tsIndex);

×

2349

  int32_t funResSize = getMaxFunResSize(pExpSup, numOfOutput);

×

2350

  if (stateType != STREAM_STATE_BUFF_HASH_SORT) {

×

2352

    pSup->pState->pResultRowStore.resultRowGet = getResultRowFromBuf;

×

2353

    pSup->pState->pResultRowStore.resultRowPut = putResultRowToBuf;

×

2354

    pSup->pState->pExprSupp = pExpSup;

×

2357

  if (stateType == STREAM_STATE_BUFF_SORT) {

×

2358

    pSup->pState->pFileState = NULL;

×

2359

    code = pSup->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SSessionKey), pSup->resultRowSize,

×

2360

                                                funResSize, sesionTs, pSup->pState, pTwAggSup->deleteMark, taskIdStr,

×

2361

                                                pHandle->checkpointId, stateType, &pSup->pState->pFileState);

×

2362

  } else if (stateType == STREAM_STATE_BUFF_HASH_SORT || stateType == STREAM_STATE_BUFF_HASH_SEARCH) {

×

2363

    pSup->pState->pFileState = NULL;

×

2364

    code = pSup->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pSup->resultRowSize, funResSize,

×

2365

                                                compareTs, pSup->pState, pTwAggSup->deleteMark, taskIdStr,

×

2366

                                                pHandle->checkpointId, stateType, &pSup->pState->pFileState);

×

2368

  QUERY_CHECK_CODE(code, lino, _end);

×

2370

  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

2371

  pSup->pResultRows = tSimpleHashInit(32, hashFn);

×

2372

  QUERY_CHECK_NULL(pSup->pResultRows, code, lino, _end, terrno);

×

2374

  for (int32_t i = 0; i < numOfOutput; ++i) {

×

2375

    pExpSup->pCtx[i].saveHandle.pState = pSup->pState;

×

2378

  pSup->pCur = NULL;

×

2380

_end:

×

2381

  if (code != TSDB_CODE_SUCCESS) {

×

2382

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

2384

  return code;

×

2387

bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap) {

×

2388

  if (ts + gap >= pWin->skey && ts - gap <= pWin->ekey) {

×

2389

    return true;

×

2391

  return false;

×

2394

bool isInWindow(SResultWindowInfo* pWinInfo, TSKEY ts, int64_t gap) {

×

2395

  return isInTimeWindow(&pWinInfo->sessionWin.win, ts, gap);

×

2398

void getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,

×

2400

  pKey->win.skey = startTs;

×

2401

  pKey->win.ekey = endTs;

×

2402

  pKey->groupId = groupId;

×

2403

  int32_t code = pAggSup->stateStore.streamStateSessionGetKeyByRange(pAggSup->pState, pKey, pKey);

×

2404

  if (code != TSDB_CODE_SUCCESS) {

×

2405

    SET_SESSION_WIN_KEY_INVALID(pKey);

×

2407

×

2409

bool isInvalidSessionWin(SResultWindowInfo* pWinInfo) { return pWinInfo->sessionWin.win.skey == 0; }

×

2411

bool inWinRange(STimeWindow* range, STimeWindow* cur) {

×

2412

  if (cur->skey >= range->skey && cur->ekey <= range->ekey) {

×

2413

    return true;

×

2415

  return false;

×

2418

void clearOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) { pAPI->streamStateClearBuff(pState, pPos); }

×

2420

int32_t setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,

×

2422

  int32_t code = TSDB_CODE_SUCCESS;

×

2423

  int32_t lino = 0;

×

2424

  pCurWin->sessionWin.groupId = groupId;

×

2425

  pCurWin->sessionWin.win.skey = startTs;

×

2426

  pCurWin->sessionWin.win.ekey = endTs;

×

2427

  int32_t size = pAggSup->resultRowSize;

×

2428

  code = pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, pAggSup->gap,

×

2429

                                                             (void**)&pCurWin->pStatePos, &size, pWinCode);

×

2430

  QUERY_CHECK_CODE(code, lino, _end);

×

2432

  if (*pWinCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->sessionWin.win)) {

×

2433

    *pWinCode = TSDB_CODE_FAILED;

×

2434

    clearOutputBuf(pAggSup->pState, pCurWin->pStatePos, &pAggSup->pSessionAPI->stateStore);

×

2437

  if (*pWinCode == TSDB_CODE_SUCCESS) {

×

2438

    pCurWin->isOutput = true;

×

2439

    if (pCurWin->pStatePos->needFree) {

×

2440

      pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->sessionWin);

×

2443

    pCurWin->sessionWin.win.skey = startTs;

×

2444

    pCurWin->sessionWin.win.ekey = endTs;

×

2446

  qDebug("===stream===set session window buff .start:%" PRId64 ",end:%" PRId64 ",groupid:%" PRIu64,

×

2448

_end:

×

2449

  if (code != TSDB_CODE_SUCCESS) {

×

2450

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

2452

  return code;

×

2455

void getSessionWinBuf(SStreamAggSupporter* pAggSup, SStreamStateCur* pCur, SResultWindowInfo* pWinInfo,

×

2457

  int32_t size = 0;

×

2458

  (*pWinCode) = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin,

×

2459

                                                                 (void**)&pWinInfo->pStatePos, &size);

×

2460

  if ((*pWinCode) != TSDB_CODE_SUCCESS) {

×

2461

    return;

×

2464

  pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pCur);

×

2467

int32_t saveDeleteInfo(SArray* pWins, SSessionKey key) {

×

2468

  int32_t code = TSDB_CODE_SUCCESS;

×

2469

  int32_t lino = 0;

×

2470

  void*   res = taosArrayPush(pWins, &key);

×

2471

  if (!res) {

×

2472

    code = terrno;

×

2473

    QUERY_CHECK_CODE(code, lino, _end);

×

2476

_end:

×

2477

  if (code != TSDB_CODE_SUCCESS) {

×

2478

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

2480

  return code;

×

2483

int32_t saveDeleteRes(SSHashObj* pStDelete, SSessionKey key) {

×

2484

  key.win.ekey = key.win.skey;

×

2485

  return tSimpleHashPut(pStDelete, &key, sizeof(SSessionKey), NULL, 0);

×

2488

void releaseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) {

×

2489

  pAPI->streamStateReleaseBuf(pState, pPos, false);

×

2490

×

2492

void removeSessionResult(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey* pKey) {

×

2493

  int32_t     code = TSDB_CODE_SUCCESS;

×

2494

  int32_t     lino = 0;

×

2495

  SSessionKey key = {0};

×

2496

  getSessionHashKey(pKey, &key);

×

2497

  void* pVal = tSimpleHashGet(pHashMap, &key, sizeof(SSessionKey));

×

2498

  if (pVal) {

×

2499

    releaseOutputBuf(pAggSup->pState, *(void**)pVal, &pAggSup->pSessionAPI->stateStore);

×

2500

    int32_t tmpRes = tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey));

×

2501

    qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

2503

  int32_t tmpRes = tSimpleHashRemove(pResMap, &key, sizeof(SSessionKey));

×

2504

  qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

2505

×

2507

void getSessionHashKey(const SSessionKey* pKey, SSessionKey* pHashKey) {

×

2508

  *pHashKey = *pKey;

×

2509

  pHashKey->win.ekey = pKey->win.skey;

×

2510

×

2512

void removeSessionDeleteResults(SSHashObj* pHashMap, SArray* pWins) {

×

2513

  if (tSimpleHashGetSize(pHashMap) == 0) {

×

2514

    return;

×

2516

  int32_t size = taosArrayGetSize(pWins);

×

2517

  for (int32_t i = 0; i < size; i++) {

×

2518

    SResultWindowInfo* pWin = taosArrayGet(pWins, i);

×

2519

    if (!pWin) continue;

×

2520

    SSessionKey key = {0};

×

2521

    getSessionHashKey(&pWin->sessionWin, &key);

×

2522

    int32_t tmpRes = tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey));

×

2523

    qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

2527

void removeSessionResults(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SArray* pWins) {

×

2528

  if (tSimpleHashGetSize(pHashMap) == 0) {

×

2529

    return;

×

2531

  int32_t size = taosArrayGetSize(pWins);

×

2532

  for (int32_t i = 0; i < size; i++) {

×

2533

    SSessionKey* pWin = taosArrayGet(pWins, i);

×

2534

    if (!pWin) continue;

×

2535

    SSessionKey key = {0};

×

2536

    getSessionHashKey(pWin, &key);

×

2537

    void* pVal = tSimpleHashGet(pHashMap, &key, sizeof(SSessionKey));

×

2538

    if (pVal) {

×

2539

      releaseOutputBuf(pAggSup->pState, *(void**)pVal, &pAggSup->pSessionAPI->stateStore);

×

2540

      int32_t tmpRes = tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey));

×

2541

      qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

2546

int32_t updateSessionWindowInfo(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo, TSKEY* pStartTs,

×

2550

  int32_t code = TSDB_CODE_SUCCESS;

×

2551

  int32_t lino = 0;

×

2552

  for (int32_t i = start; i < rows; ++i) {

×

2553

    if (!isInWindow(pWinInfo, pStartTs[i], gap) && (!pEndTs || !isInWindow(pWinInfo, pEndTs[i], gap))) {

×

2554

      (*pWinRos) = i - start;

×

2555

      goto _end;

×

2557

    if (pWinInfo->sessionWin.win.skey > pStartTs[i]) {

×

2558

      if (pStDeleted && pWinInfo->isOutput) {

×

2559

        code = saveDeleteRes(pStDeleted, pWinInfo->sessionWin);

×

2560

        QUERY_CHECK_CODE(code, lino, _end);

×

2563

      removeSessionResult(pAggSup, pStUpdated, pResultRows, &pWinInfo->sessionWin);

×

2564

      pWinInfo->sessionWin.win.skey = pStartTs[i];

×

2566

    pWinInfo->sessionWin.win.ekey = TMAX(pWinInfo->sessionWin.win.ekey, pStartTs[i]);

×

2567

    if (pEndTs) {

×

2568

      pWinInfo->sessionWin.win.ekey = TMAX(pWinInfo->sessionWin.win.ekey, pEndTs[i]);

×

2570

    memcpy(pWinInfo->pStatePos->pKey, &pWinInfo->sessionWin, sizeof(SSessionKey));

×

2572

  (*pWinRos) = rows - start;

×

2574

_end:

×

2575

  if (code != TSDB_CODE_SUCCESS) {

×

2576

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

2578

  return code;

×

2581

static int32_t initSessionOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, SqlFunctionCtx* pCtx,

×

2583

  *pResult = (SResultRow*)pWinInfo->pStatePos->pRowBuff;

×

2585

  (*pResult)->win = pWinInfo->sessionWin.win;

×

2586

  return setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowEntryInfoOffset);

×

2589

int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindowInfo* pCurWin, SResultRow** pResult,

×

2592

  int32_t        code = TSDB_CODE_SUCCESS;

×

2593

  int32_t        lino = 0;

×

2594

  SExprSupp*     pSup = &pOperator->exprSupp;

×

2595

  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

×

2596

  code = initSessionOutputBuf(pCurWin, pResult, pSup->pCtx, numOutput, pSup->rowEntryInfoOffset);

×

2597

  QUERY_CHECK_CODE(code, lino, _end);

×

2599

  updateTimeWindowInfo(pTimeWindowData, &pCurWin->sessionWin.win, winDelta);

×

2600

  code = applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, pTimeWindowData, startIndex, winRows, rows, numOutput);

×

2602

_end:

×

2603

  if (code != TSDB_CODE_SUCCESS) {

×

2604

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

2606

  return code;

×

2609

void doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey) {

×

2610

  pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, pKey);

×

2611

  SSessionKey hashKey = {0};

×

2612

  getSessionHashKey(pKey, &hashKey);

×

2613

  int32_t tmpRes = tSimpleHashRemove(pAggSup->pResultRows, &hashKey, sizeof(SSessionKey));

×

2614

  qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

2615

×

2617

void setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo* pWinInfo) {

×

2618

  void* pVal = tSimpleHashGet(pStUpdated, &pWinInfo->sessionWin, sizeof(SSessionKey));

×

2619

  if (pVal) {

×

2620

    SResultWindowInfo* pWin = pVal;

×

2621

    pWinInfo->isOutput = pWin->isOutput;

×

2623

×

2625

void getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin,

×

2627

  SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->sessionWin);

×

2628

  pNextWin->isOutput = true;

×

2629

  setSessionWinOutputInfo(pStUpdated, pNextWin);

×

2630

  int32_t size = 0;

×

2631

  pNextWin->sessionWin = pCurWin->sessionWin;

×

2632

  int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->sessionWin,

×

2633

                                                                  (void**)&pNextWin->pStatePos, &size);

×

2634

  if (code != TSDB_CODE_SUCCESS) {

×

2635

    SET_SESSION_WIN_INVALID(*pNextWin);

×

2637

  pAggSup->stateStore.streamStateFreeCur(pCur);

×

2638

×

2640

int32_t compactTimeWindow(SExprSupp* pSup, SStreamAggSupporter* pAggSup, STimeWindowAggSupp* pTwAggSup,

×

2643

  int32_t     code = TSDB_CODE_SUCCESS;

×

2644

  int32_t     lino = 0;

×

2645

  SResultRow* pCurResult = NULL;

×

2646

  int32_t     numOfOutput = pSup->numOfExprs;

×

2647

  code = initSessionOutputBuf(pCurWin, &pCurResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset);

×

2648

  QUERY_CHECK_CODE(code, lino, _end);

×

2650

  SResultRow* pWinResult = NULL;

×

2651

  code = initSessionOutputBuf(pNextWin, &pWinResult, pAggSup->pDummyCtx, numOfOutput, pSup->rowEntryInfoOffset);

×

2652

  QUERY_CHECK_CODE(code, lino, _end);

×

2654

  pCurWin->sessionWin.win.ekey = TMAX(pCurWin->sessionWin.win.ekey, pNextWin->sessionWin.win.ekey);

×

2655

  memcpy(pCurWin->pStatePos->pKey, &pCurWin->sessionWin, sizeof(SSessionKey));

×

2657

  int64_t winDelta = 0;

×

2658

  if (addGap) {

×

2659

    winDelta = pAggSup->gap;

×

2661

  updateTimeWindowInfo(&pTwAggSup->timeWindowData, &pCurWin->sessionWin.win, winDelta);

×

2662

  code = compactFunctions(pSup->pCtx, pAggSup->pDummyCtx, numOfOutput, pTaskInfo, &pTwAggSup->timeWindowData);

×

2663

  QUERY_CHECK_CODE(code, lino, _end);

×

2665

  int32_t tmpRes = tSimpleHashRemove(pStUpdated, &pNextWin->sessionWin, sizeof(SSessionKey));

×

2666

  qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

2668

  if (pNextWin->isOutput && pStDeleted) {

×

2669

    qDebug("===stream=== save delete window info %" PRId64 ", %" PRIu64, pNextWin->sessionWin.win.skey,

×

2671

    code = saveDeleteRes(pStDeleted, pNextWin->sessionWin);

×

2672

    QUERY_CHECK_CODE(code, lino, _end);

×

2674

  removeSessionResult(pAggSup, pStUpdated, pAggSup->pResultRows, &pNextWin->sessionWin);

×

2675

  doDeleteSessionWindow(pAggSup, &pNextWin->sessionWin);

×

2676

  releaseOutputBuf(pAggSup->pState, pNextWin->pStatePos, &pAggSup->pSessionAPI->stateStore);

×

2678

_end:

×

2679

  if (code != TSDB_CODE_SUCCESS) {

×

2680

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

2682

  return code;

×

2685

int32_t compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SSHashObj* pStUpdated,

×

2687

  int32_t                        code = TSDB_CODE_SUCCESS;

×

2688

  int32_t                        lino = 0;

×

2689

  SExprSupp*                     pSup = &pOperator->exprSupp;

×

2690

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

2691

  SStorageAPI*                   pAPI = &pOperator->pTaskInfo->storageAPI;

×

2692

  int32_t                        winNum = 0;

×

2693

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

2694

  SResultRow*                    pCurResult = NULL;

×

2695

  int32_t                        numOfOutput = pOperator->exprSupp.numOfExprs;

×

2696

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

2698

  if (pIsEnd != NULL) {

×

2699

    (*pIsEnd) = false;

×

2702

  while (1) {

×

2703

    SResultWindowInfo winInfo = {0};

×

2704

    getNextSessionWinInfo(pAggSup, pStUpdated, pCurWin, &winInfo);

×

2705

    if (!IS_VALID_SESSION_WIN(winInfo)) {

×

2706

      if (pIsEnd != NULL) {

×

2707

        (*pIsEnd) = true;

×

2709

      releaseOutputBuf(pAggSup->pState, winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);

×

2710

      break;

×

2712

    if (!isInWindow(pCurWin, winInfo.sessionWin.win.skey, pAggSup->gap) ||

×

2713

        !inWinRange(&pAggSup->winRange, &winInfo.sessionWin.win)) {

×

2714

      releaseOutputBuf(pAggSup->pState, winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);

×

2715

      break;

×

2718

        compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, pCurWin, &winInfo, pStUpdated, pStDeleted, true);

×

2719

    QUERY_CHECK_CODE(code, lino, _end);

×

2720

    winNum++;

×

2722

  if (pWinNum) {

×

2723

    (*pWinNum) = winNum;

×

2726

_end:

×

2727

  if (code != TSDB_CODE_SUCCESS) {

×

2728

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

2730

  return code;

×

2733

static void compactSessionSemiWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin) {

×

2734

  SExprSupp*                     pSup = &pOperator->exprSupp;

×

2735

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

2736

  SStorageAPI*                   pAPI = &pOperator->pTaskInfo->storageAPI;

×

2737

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

2738

  SResultRow*                    pCurResult = NULL;

×

2739

  int32_t                        numOfOutput = pOperator->exprSupp.numOfExprs;

×

2740

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

2742

  while (1) {

×

2743

    SResultWindowInfo winInfo = {0};

×

2744

    getNextSessionWinInfo(pAggSup, NULL, pCurWin, &winInfo);

×

2745

    if (!IS_VALID_SESSION_WIN(winInfo) || !isInWindow(pCurWin, winInfo.sessionWin.win.skey, pAggSup->gap) ||

×

2746

        !inWinRange(&pAggSup->winRange, &winInfo.sessionWin.win)) {

×

2747

      releaseOutputBuf(pAggSup->pState, winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);

×

2748

      break;

×

2750

    pCurWin->sessionWin.win.ekey = TMAX(pCurWin->sessionWin.win.ekey, winInfo.sessionWin.win.ekey);

×

2751

    memcpy(pCurWin->pStatePos->pKey, &pCurWin->sessionWin, sizeof(SSessionKey));

×

2752

    doDeleteSessionWindow(pAggSup, &winInfo.sessionWin);

×

2753

    releaseOutputBuf(pAggSup->pState, winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);

×

2755

×

2757

int32_t saveSessionOutputBuf(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo) {

×

2758

  qDebug("===stream===try save session result skey:%" PRId64 ", ekey:%" PRId64 ".pos%d", pWinInfo->sessionWin.win.skey,

×

2760

  return pAggSup->stateStore.streamStateSessionPut(pAggSup->pState, &pWinInfo->sessionWin, pWinInfo->pStatePos,

×

2764

static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pStUpdated,

×

2766

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

2767

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

2768

  int32_t                        numOfOutput = pOperator->exprSupp.numOfExprs;

×

2769

  uint64_t                       groupId = pSDataBlock->info.id.groupId;

×

2770

  int32_t                        code = TSDB_CODE_SUCCESS;

×

2771

  int32_t                        lino = 0;

×

2772

  SResultRow*                    pResult = NULL;

×

2773

  int32_t                        rows = pSDataBlock->info.rows;

×

2774

  int32_t                        winRows = 0;

×

2775

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

2777

  pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);

×

2778

  pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;

×

2779

  if (pAggSup->winRange.ekey <= 0) {

×

2780

    pAggSup->winRange.ekey = INT64_MAX;

×

2783

  SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);

×

2784

  TSKEY*           startTsCols = (int64_t*)pStartTsCol->pData;

×

2785

  SColumnInfoData* pEndTsCol = NULL;

×

2786

  if (hasEndTs) {

×

2787

    pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->endTsIndex);

×

2789

    pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);

×

2792

  TSKEY* endTsCols = (int64_t*)pEndTsCol->pData;

×

2794

  void*            pPkVal = NULL;

×

2795

  int32_t          pkLen = 0;

×

2796

  SColumnInfoData* pPkColDataInfo = NULL;

×

2797

  if (hasSrcPrimaryKeyCol(&pInfo->basic)) {

×

2798

    pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex);

×

2801

  for (int32_t i = 0; i < rows;) {

×

2802

    if (hasSrcPrimaryKeyCol(&pInfo->basic) && !IS_FINAL_SESSION_OP(pOperator) && pInfo->ignoreExpiredData) {

×

2803

      pPkVal = colDataGetData(pPkColDataInfo, i);

×

2804

      pkLen = colDataGetRowLength(pPkColDataInfo, i);

×

2806

    if (!IS_FINAL_SESSION_OP(pOperator) && pInfo->ignoreExpiredData &&

×

2807

        checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo, &pInfo->twAggSup,

×

2808

                         pSDataBlock->info.id.uid, endTsCols[i], pPkVal, pkLen)) {

×

2809

      i++;

×

2810

      continue;

×

2812

    SResultWindowInfo winInfo = {0};

×

2813

    int32_t           winCode = TSDB_CODE_SUCCESS;

×

2814

    code = setSessionOutputBuf(pAggSup, startTsCols[i], endTsCols[i], groupId, &winInfo, &winCode);

×

2815

    QUERY_CHECK_CODE(code, lino, _end);

×

2817

    if (winCode != TSDB_CODE_SUCCESS && IS_NORMAL_SESSION_OP(pOperator) &&

×

2818

        BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {

×

2819

      code = addSessionAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &winInfo.sessionWin, &pInfo->basic.notifyEventSup,

×

2821

      QUERY_CHECK_CODE(code, lino, _end);

×

2825

    if (!winInfo.pStatePos) {

×

2826

      continue;

×

2828

    setSessionWinOutputInfo(pStUpdated, &winInfo);

×

2829

    code = updateSessionWindowInfo(pAggSup, &winInfo, startTsCols, endTsCols, groupId, rows, i, pAggSup->gap,

×

2831

    QUERY_CHECK_CODE(code, lino, _end);

×

2833

    int64_t winDelta = 0;

×

2834

    if (addGap) {

×

2835

      winDelta = pAggSup->gap;

×

2837

    code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &winInfo, &pResult, i, winRows, rows, numOfOutput,

×

2839

    QUERY_CHECK_CODE(code, lino, _end);

×

2841

    code = compactSessionWindow(pOperator, &winInfo, pStUpdated, pStDeleted, addGap, NULL, NULL);

×

2842

    QUERY_CHECK_CODE(code, lino, _end);

×

2844

    code = saveSessionOutputBuf(pAggSup, &winInfo);

×

2845

    QUERY_CHECK_CODE(code, lino, _end);

×

2847

    if (pInfo->destHasPrimaryKey && winInfo.isOutput && IS_NORMAL_SESSION_OP(pOperator)) {

×

2848

      code = saveDeleteRes(pInfo->pPkDeleted, winInfo.sessionWin);

×

2849

      QUERY_CHECK_CODE(code, lino, _end);

×

2851

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) {

×

2852

      code = saveResult(winInfo, pStUpdated);

×

2853

      QUERY_CHECK_CODE(code, lino, _end);

×

2855

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

2856

      winInfo.pStatePos->beUpdated = true;

×

2857

      SSessionKey key = {0};

×

2858

      getSessionHashKey(&winInfo.sessionWin, &key);

×

2859

      code = tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo));

×

2860

      QUERY_CHECK_CODE(code, lino, _end);

×

2863

    i += winRows;

×

2866

_end:

×

2867

  if (code != TSDB_CODE_SUCCESS) {

×

2868

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

2870

×

2872

int32_t doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result) {

×

2873

  int32_t          code = TSDB_CODE_SUCCESS;

×

2874

  int32_t          lino = 0;

×

2875

  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);

×

2876

  TSKEY*           startDatas = (TSKEY*)pStartTsCol->pData;

×

2877

  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);

×

2878

  TSKEY*           endDatas = (TSKEY*)pEndTsCol->pData;

×

2879

  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);

×

2880

  uint64_t*        gpDatas = (uint64_t*)pGroupCol->pData;

×

2881

  for (int32_t i = 0; i < pBlock->info.rows; i++) {

×

2882

    while (1) {

×

2883

      SSessionKey curWin = {0};

×

2884

      getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], &curWin);

×

2885

      if (IS_INVALID_SESSION_WIN_KEY(curWin)) {

×

2886

        break;

×

2888

      doDeleteSessionWindow(pAggSup, &curWin);

×

2889

      if (result) {

×

2890

        code = saveDeleteInfo(result, curWin);

×

2891

        QUERY_CHECK_CODE(code, lino, _end);

×

2896

_end:

×

2897

  if (code != TSDB_CODE_SUCCESS) {

×

2898

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

2900

  return code;

×

2903

inline int32_t sessionKeyCompareAsc(const void* pKey1, const void* pKey2) {

×

2904

  SResultWindowInfo* pWinInfo1 = (SResultWindowInfo*)pKey1;

×

2905

  SResultWindowInfo* pWinInfo2 = (SResultWindowInfo*)pKey2;

×

2906

  SSessionKey*       pWin1 = &pWinInfo1->sessionWin;

×

2907

  SSessionKey*       pWin2 = &pWinInfo2->sessionWin;

×

2909

  if (pWin1->groupId > pWin2->groupId) {

×

2910

    return 1;

×

2911

  } else if (pWin1->groupId < pWin2->groupId) {

×

2912

    return -1;

×

2915

  if (pWin1->win.skey > pWin2->win.skey) {

×

2916

    return 1;

×

2917

  } else if (pWin1->win.skey < pWin2->win.skey) {

×

2918

    return -1;

×

2921

  return 0;

×

2924

static int32_t appendToDeleteDataBlock(SOperatorInfo* pOp, SSDataBlock *pBlock, SSessionKey *pKey) {

×

2925

  int32_t        code = TSDB_CODE_SUCCESS;

×

2926

  int32_t        lino = 0;

×

2927

  SExecTaskInfo* pTaskInfo = pOp->pTaskInfo;

×

2929

  QUERY_CHECK_NULL(pBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);

×

2930

  QUERY_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA);

×

2932

  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);

×

2933

  code = colDataSetVal(pStartTsCol, pBlock->info.rows, (const char*)&pKey->win.skey, false);

×

2934

  QUERY_CHECK_CODE(code, lino, _end);

×

2936

  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);

×

2937

  code = colDataSetVal(pEndTsCol, pBlock->info.rows, (const char*)&pKey->win.skey, false);

×

2938

  QUERY_CHECK_CODE(code, lino, _end);

×

2940

  SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);

×

2941

  colDataSetNULL(pUidCol, pBlock->info.rows);

×

2943

  SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);

×

2944

  code = colDataSetVal(pGpCol, pBlock->info.rows, (const char*)&pKey->groupId, false);

×

2945

  QUERY_CHECK_CODE(code, lino, _end);

×

2947

  SColumnInfoData* pCalStCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);

×

2948

  colDataSetNULL(pCalStCol, pBlock->info.rows);

×

2950

  SColumnInfoData* pCalEdCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);

×

2951

  colDataSetNULL(pCalEdCol, pBlock->info.rows);

×

2953

  SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);

×

2954

  if (!pTableCol) {

×

2955

    QUERY_CHECK_CODE(code, lino, _end);

×

2958

  void*        tbname = NULL;

×

2959

  int32_t      winCode = TSDB_CODE_SUCCESS;

×

2960

  SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI;

×

2962

      pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, pKey->groupId, &tbname, false, &winCode);

×

2963

  QUERY_CHECK_CODE(code, lino, _end);

×

2965

  if (winCode != TSDB_CODE_SUCCESS) {

×

2966

    colDataSetNULL(pTableCol, pBlock->info.rows);

×

2969

    STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));

×

2970

    code = colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false);

×

2971

    QUERY_CHECK_CODE(code, lino, _end);

×

2972

    pAPI->stateStore.streamStateFreeVal(tbname);

×

2974

  pBlock->info.rows += 1;

×

2976

_end:

×

2977

  if (code != TSDB_CODE_SUCCESS) {

×

2978

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

2980

  return code;

×

2983

void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite,

×

2985

  int32_t        code = TSDB_CODE_SUCCESS;

×

2986

  int32_t        lino = 0;

×

2987

  SExecTaskInfo* pTaskInfo = pOp->pTaskInfo;

×

2988

  int64_t        minWindowSize = getMinWindowSize(pOp);

×

2989

  int32_t        numOfRows = getNumOfTotalRes(pGroupResInfo);

×

2991

  blockDataCleanup(pBlock);

×

2992

  int32_t size = tSimpleHashGetSize(pStDeleted);

×

2993

  if (minWindowSize > 0) {

×

2995

    for (int32_t i = pGroupResInfo->delIndex; i < numOfRows; ++i) {

×

2996

      SResultWindowInfo* pWinInfo = taosArrayGet(pGroupResInfo->pRows, i);

×

2997

      SRowBuffPos*       pPos = pWinInfo->pStatePos;

×

2998

      SSessionKey*       pKey = (SSessionKey*)pPos->pKey;

×

2999

      if (pKey->win.ekey - pKey->win.skey < minWindowSize) {

×

3000

        size++;

×

3004

  if (size == 0) {

×

3005

    return;

×

3007

  code = blockDataEnsureCapacity(pBlock, size);

×

3008

  QUERY_CHECK_CODE(code, lino, _end);

×

3010

  int32_t iter = 0;

×

3011

  while (((*Ite) = tSimpleHashIterate(pStDeleted, *Ite, &iter)) != NULL) {

×

3012

    if (pBlock->info.rows + 1 > pBlock->info.capacity) {

×

3013

      break;

×

3015

    SSessionKey*     res = tSimpleHashGetKey(*Ite, NULL);

×

3016

    code = appendToDeleteDataBlock(pOp, pBlock, res);

×

3017

    QUERY_CHECK_CODE(code, lino, _end);

×

3020

  if (minWindowSize > 0) {

×

3021

    for (int32_t i = pGroupResInfo->delIndex; i < numOfRows; ++i) {

×

3022

      SResultWindowInfo* pWinInfo = taosArrayGet(pGroupResInfo->pRows, i);

×

3023

      SRowBuffPos*       pPos = pWinInfo->pStatePos;

×

3024

      SSessionKey*       pKey = (SSessionKey*)pPos->pKey;

×

3025

      if (pKey->win.ekey - pKey->win.skey < minWindowSize) {

×

3026

        code = appendToDeleteDataBlock(pOp, pBlock, pKey);

×

3027

        QUERY_CHECK_CODE(code, lino, _end);

×

3030

    pGroupResInfo->delIndex = numOfRows;

×

3033

_end:

×

3034

  if ((*Ite) == NULL) {

×

3035

    tSimpleHashClear(pStDeleted);

×

3038

  if (code != TSDB_CODE_SUCCESS) {

×

3039

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3043

static int32_t rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SSHashObj* pStUpdated) {

×

3044

  int32_t        winCode = TSDB_CODE_SUCCESS;

×

3045

  int32_t        code = TSDB_CODE_SUCCESS;

×

3046

  int32_t        lino = 0;

×

3047

  SExprSupp*     pSup = &pOperator->exprSupp;

×

3048

  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

×

3049

  SStorageAPI*   pAPI = &pOperator->pTaskInfo->storageAPI;

×

3051

  int32_t                        size = taosArrayGetSize(pWinArray);

×

3052

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3053

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

3054

  int32_t                        numOfOutput = pSup->numOfExprs;

×

3055

  int32_t                        numOfChild = taosArrayGetSize(pInfo->pChildren);

×

3057

  for (int32_t i = 0; i < size; i++) {

×

3058

    SSessionKey*      pWinKey = taosArrayGet(pWinArray, i);

×

3059

    int32_t           num = 0;

×

3060

    SResultWindowInfo parentWin = {0};

×

3061

    for (int32_t j = 0; j < numOfChild; j++) {

×

3062

      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, j);

×

3063

      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;

×

3064

      SStreamAggSupporter*           pChAggSup = &pChInfo->streamAggSup;

×

3065

      SSessionKey                    chWinKey = {0};

×

3066

      getSessionHashKey(pWinKey, &chWinKey);

×

3067

      SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pChAggSup->pState, &chWinKey);

×

3068

      SResultRow*      pResult = NULL;

×

3069

      SResultRow*      pChResult = NULL;

×

3070

      while (1) {

×

3071

        SResultWindowInfo childWin = {0};

×

3072

        childWin.sessionWin = *pWinKey;

×

3073

        getSessionWinBuf(pChAggSup, pCur, &childWin, &winCode);

×

3075

        if (winCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &childWin.sessionWin.win)) {

×

3076

          releaseOutputBuf(pAggSup->pState, childWin.pStatePos, &pAggSup->stateStore);

×

3077

          continue;

×

3080

        if (winCode == TSDB_CODE_SUCCESS && inWinRange(&pWinKey->win, &childWin.sessionWin.win)) {

×

3081

          if (num == 0) {

×

3082

            code = setSessionOutputBuf(pAggSup, pWinKey->win.skey, pWinKey->win.ekey, pWinKey->groupId, &parentWin,

×

3084

            QUERY_CHECK_CODE(code, lino, _end);

×

3086

            parentWin.sessionWin = childWin.sessionWin;

×

3087

            memcpy(parentWin.pStatePos->pKey, &parentWin.sessionWin, sizeof(SSessionKey));

×

3088

            code = initSessionOutputBuf(&parentWin, &pResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset);

×

3089

            QUERY_CHECK_CODE(code, lino, _end);

×

3091

          num++;

×

3092

          parentWin.sessionWin.win.skey = TMIN(parentWin.sessionWin.win.skey, childWin.sessionWin.win.skey);

×

3093

          parentWin.sessionWin.win.ekey = TMAX(parentWin.sessionWin.win.ekey, childWin.sessionWin.win.ekey);

×

3094

          memcpy(parentWin.pStatePos->pKey, &parentWin.sessionWin, sizeof(SSessionKey));

×

3096

          updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin.sessionWin.win, pAggSup->gap);

×

3097

          code = initSessionOutputBuf(&childWin, &pChResult, pChild->exprSupp.pCtx, numOfOutput,

×

3099

          QUERY_CHECK_CODE(code, lino, _end);

×

3101

          code = compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo,

×

3103

          QUERY_CHECK_CODE(code, lino, _end);

×

3105

          code = compactSessionWindow(pOperator, &parentWin, pStUpdated, NULL, true, NULL, NULL);

×

3106

          QUERY_CHECK_CODE(code, lino, _end);

×

3108

          releaseOutputBuf(pAggSup->pState, childWin.pStatePos, &pAggSup->stateStore);

×

3110

          releaseOutputBuf(pAggSup->pState, childWin.pStatePos, &pAggSup->stateStore);

×

3111

          break;

×

3114

      pAPI->stateStore.streamStateFreeCur(pCur);

×

3116

    if (num > 0) {

×

3117

      code = saveResult(parentWin, pStUpdated);

×

3118

      QUERY_CHECK_CODE(code, lino, _end);

×

3120

      code = saveSessionOutputBuf(pAggSup, &parentWin);

×

3121

      QUERY_CHECK_CODE(code, lino, _end);

×

3125

_end:

×

3126

  if (code != TSDB_CODE_SUCCESS) {

×

3127

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3129

  return code;

×

3132

int32_t closeSessionWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SSHashObj* pClosed) {

×

3133

  int32_t code = TSDB_CODE_SUCCESS;

×

3134

  int32_t lino = 0;

×

3135

  void*   pIte = NULL;

×

3136

  int32_t iter = 0;

×

3137

  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {

×

3138

    SResultWindowInfo* pWinInfo = pIte;

×

3139

    if (isCloseWindow(&pWinInfo->sessionWin.win, pTwSup)) {

×

3140

      if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE && pClosed) {

×

3141

        code = saveResult(*pWinInfo, pClosed);

×

3142

        QUERY_CHECK_CODE(code, lino, _end);

×

3144

      SSessionKey* pKey = tSimpleHashGetKey(pIte, NULL);

×

3145

      code = tSimpleHashIterateRemove(pHashMap, pKey, sizeof(SSessionKey), &pIte, &iter);

×

3146

      QUERY_CHECK_CODE(code, lino, _end);

×

3149

_end:

×

3150

  if (code != TSDB_CODE_SUCCESS) {

×

3151

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3153

  return code;

×

3156

static int32_t closeChildSessionWindow(SArray* pChildren, TSKEY maxTs) {

×

3157

  int32_t code = TSDB_CODE_SUCCESS;

×

3158

  int32_t lino = 0;

×

3160

  int32_t size = taosArrayGetSize(pChildren);

×

3161

  for (int32_t i = 0; i < size; i++) {

×

3162

    SOperatorInfo*                 pChildOp = taosArrayGetP(pChildren, i);

×

3163

    SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info;

×

3164

    pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);

×

3165

    code = closeSessionWindow(pChInfo->streamAggSup.pResultRows, &pChInfo->twAggSup, NULL);

×

3166

    QUERY_CHECK_CODE(code, lino, _end);

×

3168

_end:

×

3169

  if (code != TSDB_CODE_SUCCESS) {

×

3170

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3172

  return code;

×

3175

int32_t getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated) {

×

3176

  int32_t code = TSDB_CODE_SUCCESS;

×

3177

  int32_t lino = 0;

×

3178

  void*   pIte = NULL;

×

3179

  int32_t iter = 0;

×

3180

  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {

×

3181

    SResultWindowInfo* pWinInfo = pIte;

×

3182

    if (!pWinInfo->pStatePos->beUpdated) {

×

3183

      continue;

×

3185

    pWinInfo->pStatePos->beUpdated = false;

×

3186

    code = saveResult(*pWinInfo, pStUpdated);

×

3187

    QUERY_CHECK_CODE(code, lino, _end);

×

3190

_end:

×

3191

  if (code != TSDB_CODE_SUCCESS) {

×

3192

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3194

  return code;

×

3197

int32_t copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted) {

×

3198

  int32_t code = TSDB_CODE_SUCCESS;

×

3199

  int32_t lino = 0;

×

3200

  int32_t size = taosArrayGetSize(pResWins);

×

3201

  for (int32_t i = 0; i < size; i++) {

×

3202

    SSessionKey* pWinKey = taosArrayGet(pResWins, i);

×

3203

    if (!pWinKey) continue;

×

3204

    SSessionKey winInfo = {0};

×

3205

    getSessionHashKey(pWinKey, &winInfo);

×

3206

    code = tSimpleHashPut(pStDeleted, &winInfo, sizeof(SSessionKey), NULL, 0);

×

3207

    QUERY_CHECK_CODE(code, lino, _end);

×

3210

_end:

×

3211

  if (code != TSDB_CODE_SUCCESS) {

×

3212

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3214

  return code;

×

3218

void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList) {

×

3219

  pGroupResInfo->pRows = pArrayList;

×

3220

  pGroupResInfo->index = 0;

×

3221

  pGroupResInfo->pBuf = NULL;

×

3222

  pGroupResInfo->freeItem = false;

×

3223

  pGroupResInfo->delIndex = 0;

×

3224

×

3226

int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup,

×

3228

  int32_t         code = TSDB_CODE_SUCCESS;

×

3229

  int32_t         lino = 0;

×

3230

  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;

×

3231

  SStorageAPI*    pAPI = &pTaskInfo->storageAPI;

×

3232

  SExprInfo*      pExprInfo = pSup->pExprInfo;

×

3233

  int32_t         numOfExprs = pSup->numOfExprs;

×

3234

  int32_t*        rowEntryOffset = pSup->rowEntryInfoOffset;

×

3235

  SqlFunctionCtx* pCtx = pSup->pCtx;

×

3236

  int64_t         minWindowSize = getMinWindowSize(pOperator);

×

3238

  int32_t numOfRows = getNumOfTotalRes(pGroupResInfo);

×

3240

  for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) {

×

3241

    SResultWindowInfo* pWinInfo = taosArrayGet(pGroupResInfo->pRows, i);

×

3242

    SRowBuffPos*       pPos = pWinInfo->pStatePos;

×

3243

    SResultRow*        pRow = NULL;

×

3244

    SSessionKey*       pKey = (SSessionKey*)pPos->pKey;

×

3246

    if (pBlock->info.id.groupId == 0) {

×

3247

      pBlock->info.id.groupId = pKey->groupId;

×

3249

      void*   tbname = NULL;

×

3250

      int32_t winCode = TSDB_CODE_SUCCESS;

×

3251

      code = pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId,

×

3253

      QUERY_CHECK_CODE(code, lino, _end);

×

3255

      if (winCode != TSDB_CODE_SUCCESS) {

×

3256

        pBlock->info.parTbName[0] = 0;

×

3258

        memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);

×

3260

      pAPI->stateStore.streamStateFreeVal(tbname);

×

3263

      if (pBlock->info.id.groupId != pKey->groupId) {

×

3264

        break;

×

3268

    code = pAPI->stateStore.streamStateGetByPos(pState, pPos, (void**)&pRow);

×

3269

    QUERY_CHECK_CODE(code, lino, _end);

×

3271

    doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset);

×

3273

    if (pRow->numOfRows == 0) {

×

3274

      pGroupResInfo->index += 1;

×

3275

      continue;

×

3278

    if (pKey->win.ekey - pKey->win.skey < minWindowSize) {

×

3279

      qDebug("skip small window, groupId: %" PRId64 ", windowSize: %" PRId64 ", minWindowSize: %" PRId64, pKey->groupId,

×

3281

      pGroupResInfo->index += 1;

×

3282

      continue;

×

3285

    if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) {

×

3286

      break;

×

3289

    pGroupResInfo->index += 1;

×

3291

    for (int32_t j = 0; j < numOfExprs; ++j) {

×

3292

      int32_t slotId = pExprInfo[j].base.resSchema.slotId;

×

3294

      pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset);

×

3295

      if (pCtx[j].fpSet.finalize) {

×

3296

        int32_t tmpRes = pCtx[j].fpSet.finalize(&pCtx[j], pBlock);

×

3297

        if (TAOS_FAILED(tmpRes)) {

×

3298

          qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(tmpRes));

×

3299

          QUERY_CHECK_CODE(code, lino, _end);

×

3301

      } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) {

×

3306

        SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId);

×

3307

        char*            in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo);

×

3308

        for (int32_t k = 0; k < pRow->numOfRows; ++k) {

×

3309

          code = colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes);

×

3310

          QUERY_CHECK_CODE(code, lino, _end);

×

3315

    if (pSessionKeys) {

×

3316

      for (int32_t j = 0; j < pRow->numOfRows; ++j) {

×

3317

        const void* px = taosArrayPush(pSessionKeys, pKey);

×

3318

        QUERY_CHECK_NULL(px, code, lino, _end, terrno);

×

3322

    pBlock->info.dataLoad = 1;

×

3323

    pBlock->info.rows += pRow->numOfRows;

×

3325

  code = blockDataUpdateTsWindow(pBlock, 0);

×

3326

  QUERY_CHECK_CODE(code, lino, _end);

×

3328

_end:

×

3329

  if (code != TSDB_CODE_SUCCESS) {

×

3330

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3332

  return code;

×

3335

void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo, SSDataBlock* pBlock,

×

3337

  int32_t        code = TSDB_CODE_SUCCESS;

×

3338

  int32_t        lino = 0;

×

3339

  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

×

3341

  pBlock->info.version = pTaskInfo->version;

×

3343

  blockDataCleanup(pBlock);

×

3344

  taosArrayClear(pSessionKeys);

×

3345

  if (!hasRemainResults(pGroupResInfo)) {

×

3346

    cleanupGroupResInfo(pGroupResInfo);

×

3347

    goto _end;

×

3351

  pBlock->info.id.groupId = 0;

×

3352

  code = buildSessionResultDataBlock(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo, pSessionKeys);

×

3353

  QUERY_CHECK_CODE(code, lino, _end);

×

3355

  if (pBlock->info.rows == 0) {

×

3356

    cleanupGroupResInfo(pGroupResInfo);

×

3359

_end:

×

3360

  if (code != TSDB_CODE_SUCCESS) {

×

3361

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3363

×

3365

static int32_t buildSessionResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

3366

  int32_t                        code = TSDB_CODE_SUCCESS;

×

3367

  int32_t                        lino = 0;

×

3368

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3369

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

3370

  SOptrBasicInfo*                pBInfo = &pInfo->binfo;

×

3371

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

3372

  SStreamNotifyEventSupp*        pNotifySup = &pInfo->basic.notifyEventSup;

×

3373

  STaskNotifyEventStat*          pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;

×

3374

  bool                           addNotifyEvent = false;

×

3375

  addNotifyEvent = IS_NORMAL_SESSION_OP(pOperator) &&

×

3376

                   BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);

×

3377

  doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator, &pInfo->groupResInfo);

×

3378

  if (pInfo->pDelRes->info.rows > 0) {

×

3379

    printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

3380

    if (addNotifyEvent) {

×

3381

      code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);

×

3382

      QUERY_CHECK_CODE(code, lino, _end);

×

3384

    (*ppRes) = pInfo->pDelRes;

×

3385

    return code;

×

3388

  doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes,

×

3390

  if (pBInfo->pRes->info.rows > 0) {

×

3391

    printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

3392

    if (addNotifyEvent) {

×

3394

      for (int32_t i = 0; i < taosArrayGetSize(pNotifySup->pSessionKeys); ++i) {

×

3395

        SSessionKey* pKey = taosArrayGet(pNotifySup->pSessionKeys, i);

×

3396

        pKey->win.ekey += pAggSup->gap;

×

3398

      code = addAggResultNotifyEvent(pBInfo->pRes, pNotifySup->pSessionKeys, pTaskInfo->streamInfo.notifyResultSchema,

×

3400

      QUERY_CHECK_CODE(code, lino, _end);

×

3402

    (*ppRes) = pBInfo->pRes;

×

3403

    return code;

×

3406

  code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);

×

3407

  QUERY_CHECK_CODE(code, lino, _end);

×

3408

  if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {

×

3409

    printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

3410

    (*ppRes) = pNotifySup->pEventBlock;

×

3411

    return code;

×

3414

  code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);

×

3415

  QUERY_CHECK_CODE(code, lino, _end);

×

3417

_end:

×

3418

  if (code != TSDB_CODE_SUCCESS) {

×

3419

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3421

  (*ppRes) = NULL;

×

3422

  return code;

×

3425

int32_t getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins) {

×

3426

  int32_t code = TSDB_CODE_SUCCESS;

×

3427

  int32_t lino = 0;

×

3428

  int32_t size = taosArrayGetSize(pAllWins);

×

3429

  if (size == 0) {

×

3430

    goto _end;

×

3432

  SResultWindowInfo* pWinInfo = taosArrayGet(pAllWins, size - 1);

×

3433

  SSessionKey*       pSeKey = &pWinInfo->sessionWin;

×

3434

  void*              tmp = taosArrayPush(pMaxWins, pSeKey);

×

3435

  if (!tmp) {

×

3436

    code = terrno;

×

3437

    QUERY_CHECK_CODE(code, lino, _end);

×

3440

  if (pSeKey->groupId == 0) {

×

3441

    goto _end;

×

3443

  uint64_t preGpId = pSeKey->groupId;

×

3444

  for (int32_t i = size - 2; i >= 0; i--) {

×

3445

    pWinInfo = taosArrayGet(pAllWins, i);

×

3446

    pSeKey = &pWinInfo->sessionWin;

×

3447

    if (preGpId != pSeKey->groupId) {

×

3448

      void* tmp = taosArrayPush(pMaxWins, pSeKey);

×

3449

      if (!tmp) {

×

3450

        code = terrno;

×

3451

        QUERY_CHECK_CODE(code, lino, _end);

×

3453

      preGpId = pSeKey->groupId;

×

3457

_end:

×

3458

  if (code != TSDB_CODE_SUCCESS) {

×

3459

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3461

  return code;

×

3464

int32_t encodeSSessionKey(void** buf, SSessionKey* key) {

×

3465

  int32_t tlen = 0;

×

3466

  tlen += encodeSTimeWindow(buf, &key->win);

×

3467

  tlen += taosEncodeFixedU64(buf, key->groupId);

×

3468

  return tlen;

×

3471

void* decodeSSessionKey(void* buf, SSessionKey* key) {

×

3472

  buf = decodeSTimeWindow(buf, &key->win);

×

3473

  buf = taosDecodeFixedU64(buf, &key->groupId);

×

3474

  return buf;

×

3477

int32_t encodeSResultWindowInfo(void** buf, SResultWindowInfo* key, int32_t outLen) {

×

3478

  int32_t tlen = 0;

×

3479

  tlen += taosEncodeFixedBool(buf, key->isOutput);

×

3480

  tlen += encodeSSessionKey(buf, &key->sessionWin);

×

3481

  return tlen;

×

3484

void* decodeSResultWindowInfo(void* buf, SResultWindowInfo* key, int32_t outLen) {

×

3485

  buf = taosDecodeFixedBool(buf, &key->isOutput);

×

3486

  buf = decodeSSessionKey(buf, &key->sessionWin);

×

3487

  return buf;

×

3490

int32_t doStreamSessionEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOperator, bool isParent) {

×

3491

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3492

  if (!pInfo) {

×

3493

    return 0;

×

3496

  void* pData = (buf == NULL) ? NULL : *buf;

×

3499

  int32_t tlen = 0;

×

3500

  int32_t mapSize = tSimpleHashGetSize(pInfo->streamAggSup.pResultRows);

×

3501

  tlen += taosEncodeFixedI32(buf, mapSize);

×

3502

  void*   pIte = NULL;

×

3503

  size_t  keyLen = 0;

×

3504

  int32_t iter = 0;

×

3505

  while ((pIte = tSimpleHashIterate(pInfo->streamAggSup.pResultRows, pIte, &iter)) != NULL) {

×

3506

    void* key = tSimpleHashGetKey(pIte, &keyLen);

×

3507

    tlen += encodeSSessionKey(buf, key);

×

3508

    tlen += encodeSResultWindowInfo(buf, pIte, pInfo->streamAggSup.resultRowSize);

×

3512

  tlen += encodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);

×

3515

  int32_t size = taosArrayGetSize(pInfo->pChildren);

×

3516

  tlen += taosEncodeFixedI32(buf, size);

×

3517

  for (int32_t i = 0; i < size; i++) {

×

3518

    SOperatorInfo* pChOp = taosArrayGetP(pInfo->pChildren, i);

×

3519

    tlen += doStreamSessionEncodeOpState(buf, 0, pChOp, false);

×

3523

  tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);

×

3526

  tlen += encodeStreamBasicInfo(buf, &pInfo->basic);

×

3529

  if (isParent) {

×

3530

    if (buf) {

×

3531

      uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));

×

3532

      tlen += taosEncodeFixedU32(buf, cksum);

×

3534

      tlen += sizeof(uint32_t);

×

3538

  return tlen;

×

3541

int32_t doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator, bool isParent, void** ppBuf) {

×

3542

  int32_t                        code = TSDB_CODE_SUCCESS;

×

3543

  int32_t                        lino = 0;

×

3544

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3545

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

3546

  void*                        pDataEnd = POINTER_SHIFT(buf, len);

×

3547

  if (!pInfo) {

×

3548

    code = TSDB_CODE_FAILED;

×

3549

    QUERY_CHECK_CODE(code, lino, _end);

×

3551

  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;

×

3554

  if (isParent) {

×

3555

    int32_t dataLen = len - sizeof(uint32_t);

×

3556

    void*   pCksum = POINTER_SHIFT(buf, dataLen);

×

3557

    if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) {

×

3558

      qError("stream session state is invalid");

×

3559

      code = TSDB_CODE_FAILED;

×

3560

      QUERY_CHECK_CODE(code, lino, _end);

×

3562

    pDataEnd = pCksum;

×

3566

  int32_t mapSize = 0;

×

3567

  buf = taosDecodeFixedI32(buf, &mapSize);

×

3568

  for (int32_t i = 0; i < mapSize; i++) {

×

3569

    SResultWindowInfo winfo = {0};

×

3570

    buf = decodeSSessionKey(buf, &winfo.sessionWin);

×

3571

    int32_t winCode = TSDB_CODE_SUCCESS;

×

3572

    code = pAggSup->stateStore.streamStateSessionAddIfNotExist(

×

3574

    QUERY_CHECK_CODE(code, lino, _end);

×

3575

    QUERY_CHECK_CONDITION((winCode == TSDB_CODE_SUCCESS), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);

×

3577

    buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize);

×

3578

    code = tSimpleHashPut(pInfo->streamAggSup.pResultRows, &winfo.sessionWin, sizeof(SSessionKey), &winfo,

×

3580

    QUERY_CHECK_CODE(code, lino, _end);

×

3584

  buf = decodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);

×

3587

  int32_t size = 0;

×

3588

  buf = taosDecodeFixedI32(buf, &size);

×

3589

  for (int32_t i = 0; i < size; i++) {

×

3590

    SOperatorInfo* pChOp = taosArrayGetP(pInfo->pChildren, i);

×

3591

    code = doStreamSessionDecodeOpState(buf, 0, pChOp, false, &buf);

×

3592

    QUERY_CHECK_CODE(code, lino, _end);

×

3596

  buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);

×

3597

  if (ppBuf) {

×

3598

    (*ppBuf) = buf;

×

3602

  if (buf < pDataEnd) {

×

3603

    code = decodeStreamBasicInfo(&buf, &pInfo->basic);

×

3604

    QUERY_CHECK_CODE(code, lino, _end);

×

3607

_end:

×

3608

  if (code != TSDB_CODE_SUCCESS) {

×

3609

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3611

  return code;

×

3614

void doStreamSessionSaveCheckpoint(SOperatorInfo* pOperator) {

×

3615

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3616

  if (needSaveStreamOperatorInfo(&pInfo->basic)) {

×

3617

    int32_t len = doStreamSessionEncodeOpState(NULL, 0, pOperator, true);

×

3618

    void*   buf = taosMemoryCalloc(1, len);

×

3619

    if (!buf) {

×

3620

      qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY));

×

3621

      return;

×

3623

    void* pBuf = buf;

×

3624

    len = doStreamSessionEncodeOpState(&pBuf, len, pOperator, true);

×

3625

    pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME,

×

3627

    taosMemoryFree(buf);

×

3628

    saveStreamOperatorStateComplete(&pInfo->basic);

×

3632

void resetUnCloseSessionWinInfo(SSHashObj* winMap) {

×

3633

  void*   pIte = NULL;

×

3634

  int32_t iter = 0;

×

3635

  while ((pIte = tSimpleHashIterate(winMap, pIte, &iter)) != NULL) {

×

3636

    SResultWindowInfo* pResInfo = pIte;

×

3637

    pResInfo->pStatePos->beUsed = true;

×

3639

×

3641

int32_t copyDeleteSessionKey(SSHashObj* source, SSHashObj* dest) {

×

3642

  int32_t code = TSDB_CODE_SUCCESS;

×

3643

  int32_t lino = 0;

×

3644

  if (tSimpleHashGetSize(source) == 0) {

×

3645

    goto _end;

×

3647

  void*   pIte = NULL;

×

3648

  int32_t iter = 0;

×

3649

  size_t  keyLen = 0;

×

3650

  while ((pIte = tSimpleHashIterate(source, pIte, &iter)) != NULL) {

×

3651

    SSessionKey* pKey = tSimpleHashGetKey(pIte, &keyLen);

×

3652

    code = saveDeleteRes(dest, *pKey);

×

3653

    QUERY_CHECK_CODE(code, lino, _end);

×

3655

  tSimpleHashClear(source);

×

3657

_end:

×

3658

  if (code != TSDB_CODE_SUCCESS) {

×

3659

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3661

  return code;

×

3664

static int32_t doStreamSessionAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

3665

  int32_t                        code = TSDB_CODE_SUCCESS;

×

3666

  int32_t                        lino = 0;

×

3667

  SExprSupp*                     pSup = &pOperator->exprSupp;

×

3668

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3669

  SOptrBasicInfo*                pBInfo = &pInfo->binfo;

×

3670

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

3671

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

3672

  qDebug("stask:%s  %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);

×

3673

  if (pOperator->status == OP_EXEC_DONE) {

×

3674

    (*ppRes) = NULL;

×

3675

    return code;

×

3676

  } else if (pOperator->status == OP_RES_TO_RETURN) {

×

3677

    SSDataBlock* opRes = NULL;

×

3678

    code = buildSessionResult(pOperator, &opRes);

×

3679

    QUERY_CHECK_CODE(code, lino, _end);

×

3680

    if (opRes) {

×

3681

      (*ppRes) = opRes;

×

3682

      return code;

×

3685

    if (pInfo->recvGetAll) {

×

3686

      pInfo->recvGetAll = false;

×

3687

      resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows);

×

3690

    if (pInfo->reCkBlock) {

×

3691

      pInfo->reCkBlock = false;

×

3692

      printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

3693

      (*ppRes) = pInfo->pCheckpointRes;

×

3694

      return code;

×

3697

    setStreamOperatorCompleted(pOperator);

×

3698

    (*ppRes) = NULL;

×

3699

    return code;

×

3702

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

3703

  if (!pInfo->pUpdated) {

×

3704

    pInfo->pUpdated = taosArrayInit(16, sizeof(SResultWindowInfo));

×

3705

    QUERY_CHECK_NULL(pInfo->pUpdated, code, lino, _end, terrno);

×

3707

  if (!pInfo->pStUpdated) {

×

3708

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

3709

    pInfo->pStUpdated = tSimpleHashInit(64, hashFn);

×

3710

    QUERY_CHECK_NULL(pInfo->pStUpdated, code, lino, _end, terrno);

×

3712

  while (1) {

×

3713

    SSDataBlock* pBlock = NULL;

×

3714

    code = downstream->fpSet.getNextFn(downstream, &pBlock);

×

3715

    QUERY_CHECK_CODE(code, lino, _end);

×

3717

    if (pBlock == NULL) {

×

3718

      break;

×

3720

    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));

×

3721

    setStreamOperatorState(&pInfo->basic, pBlock->info.type);

×

3723

    if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||

×

3724

        pBlock->info.type == STREAM_CLEAR) {

×

3725

      SArray* pWins = taosArrayInit(16, sizeof(SSessionKey));

×

3726

      QUERY_CHECK_NULL(pWins, code, lino, _end, terrno);

×

3728

      code = doDeleteTimeWindows(pAggSup, pBlock, pWins);

×

3729

      QUERY_CHECK_CODE(code, lino, _end);

×

3731

      removeSessionResults(pAggSup, pInfo->pStUpdated, pWins);

×

3732

      if (IS_FINAL_SESSION_OP(pOperator)) {

×

3733

        int32_t                        childIndex = getChildIndex(pBlock);

×

3734

        SOperatorInfo*                 pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);

×

3735

        SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;

×

3737

        code = doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, NULL);

×

3738

        QUERY_CHECK_CODE(code, lino, _end);

×

3740

        code = rebuildSessionWindow(pOperator, pWins, pInfo->pStUpdated);

×

3741

        QUERY_CHECK_CODE(code, lino, _end);

×

3743

      code = copyDeleteWindowInfo(pWins, pInfo->pStDeleted);

×

3744

      QUERY_CHECK_CODE(code, lino, _end);

×

3746

      if (pInfo->destHasPrimaryKey && IS_NORMAL_SESSION_OP(pOperator)) {

×

3747

        code = copyDeleteWindowInfo(pWins, pInfo->pPkDeleted);

×

3748

        QUERY_CHECK_CODE(code, lino, _end);

×

3750

      taosArrayDestroy(pWins);

×

3751

      continue;

×

3752

    } else if (pBlock->info.type == STREAM_GET_ALL) {

×

3753

      pInfo->recvGetAll = true;

×

3754

      code = getAllSessionWindow(pAggSup->pResultRows, pInfo->pStUpdated);

×

3755

      QUERY_CHECK_CODE(code, lino, _end);

×

3756

      continue;

×

3757

    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {

×

3758

      (*ppRes) = pBlock;

×

3759

      return code;

×

3760

    } else if (pBlock->info.type == STREAM_CHECKPOINT) {

×

3761

      pAggSup->stateStore.streamStateCommit(pAggSup->pState);

×

3762

      doStreamSessionSaveCheckpoint(pOperator);

×

3763

      code = copyDataBlock(pInfo->pCheckpointRes, pBlock);

×

3764

      QUERY_CHECK_CODE(code, lino, _end);

×

3766

      continue;

×

3768

      if (pBlock->info.type != STREAM_NORMAL && pBlock->info.type != STREAM_INVALID) {

×

3769

        code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;

×

3770

        QUERY_CHECK_CODE(code, lino, _end);

×

3774

    if (pInfo->scalarSupp.pExprInfo != NULL) {

×

3775

      SExprSupp* pExprSup = &pInfo->scalarSupp;

×

3776

      code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);

×

3777

      QUERY_CHECK_CODE(code, lino, _end);

×

3780

    code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

3781

    QUERY_CHECK_CODE(code, lino, _end);

×

3783

    doStreamSessionAggImpl(pOperator, pBlock, pInfo->pStUpdated, pInfo->pStDeleted, IS_FINAL_SESSION_OP(pOperator),

×

3785

    if (IS_FINAL_SESSION_OP(pOperator)) {

×

3786

      int32_t chIndex = getChildIndex(pBlock);

×

3787

      int32_t size = taosArrayGetSize(pInfo->pChildren);

×

3789

      for (int32_t i = 0; i < chIndex + 1 - size; i++) {

×

3790

        SOperatorInfo* pChildOp = NULL;

×

3791

        code = createStreamFinalSessionAggOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0, NULL, &pChildOp);

×

3792

        if (pChildOp == NULL || code != 0) {

×

3793

          qError("%s create stream child of final session error", GET_TASKID(pTaskInfo));

×

3794

          code = TSDB_CODE_FAILED;

×

3795

          QUERY_CHECK_CODE(code, lino, _end);

×

3798

        void* tmp = taosArrayPush(pInfo->pChildren, &pChildOp);

×

3799

        if (!tmp) {

×

3800

          code = terrno;

×

3801

          QUERY_CHECK_CODE(code, lino, _end);

×

3805

      SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex);

×

3806

      code = setInputDataBlock(&pChildOp->exprSupp, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

3807

      QUERY_CHECK_CODE(code, lino, _end);

×

3808

      doStreamSessionAggImpl(pChildOp, pBlock, NULL, NULL, true, false);

×

3810

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);

×

3811

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.watermark);

×

3814

  pOperator->status = OP_RES_TO_RETURN;

×

3816

  code = closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pInfo->pStUpdated);

×

3817

  QUERY_CHECK_CODE(code, lino, _end);

×

3819

  code = closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs);

×

3820

  QUERY_CHECK_CODE(code, lino, _end);

×

3822

  code = copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc);

×

3823

  QUERY_CHECK_CODE(code, lino, _end);

×

3825

  if (!pInfo->destHasPrimaryKey) {

×

3826

    removeSessionDeleteResults(pInfo->pStDeleted, pInfo->pUpdated);

×

3828

  if (pInfo->isHistoryOp) {

×

3829

    code = getMaxTsWins(pInfo->pUpdated, pInfo->historyWins);

×

3830

    QUERY_CHECK_CODE(code, lino, _end);

×

3832

  if (pInfo->destHasPrimaryKey && IS_NORMAL_SESSION_OP(pOperator)) {

×

3833

    code = copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pStDeleted);

×

3834

    QUERY_CHECK_CODE(code, lino, _end);

×

3836

  initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);

×

3837

  pInfo->pUpdated = NULL;

×

3838

  code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);

×

3839

  QUERY_CHECK_CODE(code, lino, _end);

×

3841

  SSDataBlock* opRes = NULL;

×

3842

  code = buildSessionResult(pOperator, &opRes);

×

3843

  QUERY_CHECK_CODE(code, lino, _end);

×

3844

  if (opRes) {

×

3845

    (*ppRes) = opRes;

×

3846

    return code;

×

3849

_end:

×

3850

  if (code != TSDB_CODE_SUCCESS) {

×

3851

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3852

    pTaskInfo->code = code;

×

3853

    T_LONG_JMP(pTaskInfo->env, code);

×

3855

  setStreamOperatorCompleted(pOperator);

×

3856

  (*ppRes) = NULL;

×

3857

  return code;

×

3860

static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {

×

3861

  SSDataBlock* pRes = NULL;

×

3862

  int32_t      code = doStreamSessionAggNext(pOperator, &pRes);

×

3863

  return pRes;

×

3866

void streamSessionReleaseState(SOperatorInfo* pOperator) {

×

3867

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3868

  int32_t                        winSize = taosArrayGetSize(pInfo->historyWins) * sizeof(SSessionKey);

×

3869

  int32_t                        resSize = winSize + sizeof(TSKEY);

×

3870

  char*                          pBuff = taosMemoryCalloc(1, resSize);

×

3871

  if (!pBuff) {

×

3872

    return;

×

3874

  memcpy(pBuff, pInfo->historyWins->pData, winSize);

×

3875

  memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY));

×

3876

  pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_STATE_NAME,

×

3878

  pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState);

×

3879

  taosMemoryFreeClear(pBuff);

×

3880

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

3881

  if (downstream->fpSet.releaseStreamStateFn) {

×

3882

    downstream->fpSet.releaseStreamStateFn(downstream);

×

3886

void resetWinRange(STimeWindow* winRange) {

×

3887

  winRange->skey = INT64_MIN;

×

3888

  winRange->ekey = INT64_MAX;

×

3889

×

3891

int32_t getSessionWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SResultWindowInfo* pWinInfo) {

×

3892

  int32_t code = TSDB_CODE_SUCCESS;

×

3893

  int32_t lino = 0;

×

3894

  int32_t rowSize = pAggSup->resultRowSize;

×

3895

  int32_t winCode = TSDB_CODE_SUCCESS;

×

3896

  code = pAggSup->stateStore.streamStateSessionGet(pAggSup->pState, pKey, (void**)&pWinInfo->pStatePos, &rowSize,

×

3898

  QUERY_CHECK_CODE(code, lino, _end);

×

3900

  if (winCode == TSDB_CODE_SUCCESS) {

×

3901

    pWinInfo->sessionWin = *pKey;

×

3902

    pWinInfo->isOutput = true;

×

3903

    if (pWinInfo->pStatePos->needFree) {

×

3904

      pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pWinInfo->sessionWin);

×

3907

    SET_SESSION_WIN_INVALID((*pWinInfo));

×

3910

_end:

×

3911

  if (code != TSDB_CODE_SUCCESS) {

×

3912

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3914

  return code;

×

3917

void reloadAggSupFromDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup) {

×

3918

  SStateStore* pAPI = &downstream->pTaskInfo->storageAPI.stateStore;

×

3920

  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {

×

3921

    reloadAggSupFromDownStream(downstream->pDownstream[0], pAggSup);

×

3922

    return;

×

3925

  SStreamScanInfo* pScanInfo = downstream->info;

×

3926

  pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo;

×

3929

void streamSessionSemiReloadState(SOperatorInfo* pOperator) {

×

3930

  int32_t                        code = TSDB_CODE_SUCCESS;

×

3931

  int32_t                        lino = 0;

×

3932

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3933

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

3934

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

3935

  resetWinRange(&pAggSup->winRange);

×

3937

  SResultWindowInfo winInfo = {0};

×

3938

  int32_t           size = 0;

×

3939

  void*             pBuf = NULL;

×

3940

  code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME,

×

3942

  QUERY_CHECK_CODE(code, lino, _end);

×

3944

  int32_t      num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);

×

3945

  SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;

×

3946

  for (int32_t i = 0; i < num; i++) {

×

3947

    SResultWindowInfo winInfo = {0};

×

3948

    code = getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &winInfo);

×

3949

    QUERY_CHECK_CODE(code, lino, _end);

×

3950

    if (!IS_VALID_SESSION_WIN(winInfo)) {

×

3951

      continue;

×

3953

    compactSessionSemiWindow(pOperator, &winInfo);

×

3954

    code = saveSessionOutputBuf(pAggSup, &winInfo);

×

3955

    QUERY_CHECK_CODE(code, lino, _end);

×

3957

  TSKEY ts = *(TSKEY*)((char*)pBuf + size - sizeof(TSKEY));

×

3958

  taosMemoryFree(pBuf);

×

3959

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);

×

3960

  pAggSup->stateStore.streamStateReloadInfo(pAggSup->pState, ts);

×

3962

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

3963

  if (downstream->fpSet.reloadStreamStateFn) {

×

3964

    downstream->fpSet.reloadStreamStateFn(downstream);

×

3966

  reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup);

×

3968

_end:

×

3969

  if (code != TSDB_CODE_SUCCESS) {

×

3970

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3972

×

3974

void streamSessionReloadState(SOperatorInfo* pOperator) {

×

3975

  int32_t                        code = TSDB_CODE_SUCCESS;

×

3976

  int32_t                        lino = 0;

×

3977

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3978

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

3979

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

3980

  resetWinRange(&pAggSup->winRange);

×

3982

  int32_t size = 0;

×

3983

  void*   pBuf = NULL;

×

3984

  code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME,

×

3987

  QUERY_CHECK_CODE(code, lino, _end);

×

3989

  int32_t      num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);

×

3990

  SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;

×

3992

  TSKEY ts = *(TSKEY*)((char*)pBuf + size - sizeof(TSKEY));

×

3993

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);

×

3994

  pAggSup->stateStore.streamStateReloadInfo(pAggSup->pState, ts);

×

3996

  if (!pInfo->pStUpdated && num > 0) {

×

3997

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

3998

    pInfo->pStUpdated = tSimpleHashInit(64, hashFn);

×

3999

    QUERY_CHECK_NULL(pInfo->pStUpdated, code, lino, _end, terrno);

×

4001

  for (int32_t i = 0; i < num; i++) {

×

4002

    SResultWindowInfo winInfo = {0};

×

4003

    code = getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &winInfo);

×

4004

    QUERY_CHECK_CODE(code, lino, _end);

×

4005

    if (!IS_VALID_SESSION_WIN(winInfo)) {

×

4006

      continue;

×

4009

    int32_t winNum = 0;

×

4010

    code = compactSessionWindow(pOperator, &winInfo, pInfo->pStUpdated, pInfo->pStDeleted, true, &winNum, NULL);

×

4011

    QUERY_CHECK_CODE(code, lino, _end);

×

4013

    if (winNum > 0) {

×

4014

      qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, winInfo.sessionWin.win.skey,

×

4016

      if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {

×

4017

        code = saveResult(winInfo, pInfo->pStUpdated);

×

4018

        QUERY_CHECK_CODE(code, lino, _end);

×

4019

      } else if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

4020

        if (!isCloseWindow(&winInfo.sessionWin.win, &pInfo->twAggSup)) {

×

4021

          code = saveDeleteRes(pInfo->pStDeleted, winInfo.sessionWin);

×

4022

          QUERY_CHECK_CODE(code, lino, _end);

×

4024

        SSessionKey key = {0};

×

4025

        getSessionHashKey(&winInfo.sessionWin, &key);

×

4026

        code = tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo));

×

4027

        QUERY_CHECK_CODE(code, lino, _end);

×

4030

    code = saveSessionOutputBuf(pAggSup, &winInfo);

×

4031

    QUERY_CHECK_CODE(code, lino, _end);

×

4033

  taosMemoryFree(pBuf);

×

4035

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

4036

  if (downstream->fpSet.reloadStreamStateFn) {

×

4037

    downstream->fpSet.reloadStreamStateFn(downstream);

×

4039

  reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup);

×

4041

_end:

×

4042

  if (code != TSDB_CODE_SUCCESS) {

×

4043

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

4045

×

4047

int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,

×

4049

  QRY_PARAM_CHECK(pOptrInfo);

×

4051

  SSessionWinodwPhysiNode*       pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;

×

4052

  int32_t                        numOfCols = 0;

×

4053

  int32_t                        code = TSDB_CODE_OUT_OF_MEMORY;

×

4054

  int32_t                        lino = 0;

×

4055

  SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo));

×

4056

  SOperatorInfo*                 pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));

×

4057

  if (pInfo == NULL || pOperator == NULL) {

×

4058

    code = terrno;

×

4059

    goto _error;

×

4062

  pOperator->pTaskInfo = pTaskInfo;

×

4064

  initResultSizeInfo(&pOperator->resultInfo, 4096);

×

4065

  if (pSessionNode->window.pExprs != NULL) {

×

4066

    int32_t    numOfScalar = 0;

×

4067

    SExprInfo* pScalarExprInfo = NULL;

×

4068

    code = createExprInfo(pSessionNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalar);

×

4069

    QUERY_CHECK_CODE(code, lino, _error);

×

4071

    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);

×

4072

    if (code != TSDB_CODE_SUCCESS) {

×

4073

      goto _error;

×

4076

  SExprSupp* pExpSup = &pOperator->exprSupp;

×

4078

  SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);

×

4079

  QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);

×

4080

  pInfo->binfo.pRes = pResBlock;

×

4082

  SExprInfo* pExprInfo = NULL;

×

4083

  code = createExprInfo(pSessionNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);

×

4084

  QUERY_CHECK_CODE(code, lino, _error);

×

4086

  code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore);

×

4087

  QUERY_CHECK_CODE(code, lino, _error);

×

4089

  pInfo->twAggSup = (STimeWindowAggSupp){

×

4090

      .waterMark = pSessionNode->window.watermark,

×

4091

      .calTrigger = pSessionNode->window.triggerType,

×

4094

      .deleteMark = getDeleteMark(&pSessionNode->window, 0),

×

4097

  pInfo->primaryTsIndex = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;

×

4099

      initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, pSessionNode->gap, pTaskInfo->streamInfo.pState,

×

4100

                             0, 0, &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup, GET_TASKID(pTaskInfo),

×

4102

  if (code != TSDB_CODE_SUCCESS) {

×

4103

    goto _error;

×

4106

  code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

×

4107

  QUERY_CHECK_CODE(code, lino, _error);

×

4109

  if (pSessionNode->window.pTsEnd) {

×

4110

    pInfo->endTsIndex = ((SColumnNode*)pSessionNode->window.pTsEnd)->slotId;

×

4113

  pInfo->order = TSDB_ORDER_ASC;

×

4114

  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

4115

  pInfo->pStDeleted = tSimpleHashInit(64, hashFn);

×

4116

  QUERY_CHECK_NULL(pInfo->pStDeleted, code, lino, _error, terrno);

×

4117

  pInfo->pDelIterator = NULL;

×

4118

  code = createSpecialDataBlock(STREAM_DELETE_RESULT, &pInfo->pDelRes);

×

4119

  QUERY_CHECK_CODE(code, lino, _error);

×

4121

  pInfo->pChildren = NULL;

×

4122

  pInfo->pPhyNode = pPhyNode;

×

4123

  pInfo->ignoreExpiredData = pSessionNode->window.igExpired;

×

4124

  pInfo->ignoreExpiredDataSaved = false;

×

4125

  pInfo->pUpdated = NULL;

×

4126

  pInfo->pStUpdated = NULL;

×

4127

  pInfo->dataVersion = 0;

×

4128

  pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey));

×

4129

  if (!pInfo->historyWins) {

×

4130

    goto _error;

×

4132

  if (pHandle) {

×

4133

    pInfo->isHistoryOp = (pHandle->fillHistory == STREAM_HISTORY_OPERATOR);

×

4136

  code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);

×

4137

  QUERY_CHECK_CODE(code, lino, _error);

×

4139

  pInfo->clearState = false;

×

4140

  pInfo->recvGetAll = false;

×

4141

  pInfo->destHasPrimaryKey = pSessionNode->window.destHasPrimaryKey;

×

4142

  pInfo->pPkDeleted = tSimpleHashInit(64, hashFn);

×

4143

  QUERY_CHECK_NULL(pInfo->pPkDeleted, code, lino, _error, terrno);

×

4144

  pInfo->pOperator = pOperator;

×

4145

  initNonBlockAggSupptor(&pInfo->nbSup, NULL, NULL);

×

4147

  setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), nodeType(pSessionNode), true,

×

4149

  if (pPhyNode->type != QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {

×

4151

    void*   buff = NULL;

×

4152

    int32_t len = 0;

×

4154

        pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME,

×

4156

    if (res == TSDB_CODE_SUCCESS) {

×

4157

      code = doStreamSessionDecodeOpState(buff, len, pOperator, true, NULL);

×

4158

      taosMemoryFree(buff);

×

4159

      QUERY_CHECK_CODE(code, lino, _error);

×

4163

  code = initStreamBasicInfo(&pInfo->basic, pOperator);

×

4164

  QUERY_CHECK_CODE(code, lino, _error);

×

4166

  if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {

×

4167

    if (pHandle->fillHistory == STREAM_HISTORY_OPERATOR) {

×

4168

      setFillHistoryOperatorFlag(&pInfo->basic);

×

4169

    } else if (pHandle->fillHistory == STREAM_RECALCUL_OPERATOR) {

×

4170

      setRecalculateOperatorFlag(&pInfo->basic);

×

4173

        createOperatorFpSet(optrDummyOpenFn, doStreamSessionNonblockAggNext, NULL, destroyStreamSessionAggOperatorInfo,

×

4175

    setOperatorStreamStateFn(pOperator, streamSessionNonblockReleaseState, streamSessionNonblockReloadState);

×

4178

        createOperatorFpSet(optrDummyOpenFn, doStreamSessionAggNext, NULL, destroyStreamSessionAggOperatorInfo,

×

4180

    setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState);

×

4183

  if (downstream) {

×

4184

    pInfo->basic.primaryPkIndex = -1;

×

4185

    code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,

×

4186

                          &pInfo->twAggSup, &pInfo->basic, pSessionNode->window.recalculateInterval);

×

4187

    QUERY_CHECK_CODE(code, lino, _error);

×

4189

    code = appendDownstream(pOperator, &downstream, 1);

×

4190

    QUERY_CHECK_CODE(code, lino, _error);

×

4193

  *pOptrInfo = pOperator;

×

4194

  return TSDB_CODE_SUCCESS;

×

4196

_error:

×

4197

  if (pInfo != NULL) {

×

4198

    destroyStreamSessionAggOperatorInfo(pInfo);

×

4200

  destroyOperatorAndDownstreams(pOperator, &downstream, 1);

×

4201

  pTaskInfo->code = code;

×

4202

  qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

4203

  return code;

×

4206

static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) {

×

4207

  tSimpleHashClear(pInfo->streamAggSup.pResultRows);

×

4208

  pInfo->streamAggSup.stateStore.streamStateSessionClear(pInfo->streamAggSup.pState);

×

4209

  pInfo->clearState = false;

×

4210

×

4212

int32_t deleteSessionWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate,

×

4214

  int32_t code = TSDB_CODE_SUCCESS;

×

4215

  int32_t lino = 0;

×

4216

  SArray* pWins = taosArrayInit(16, sizeof(SSessionKey));

×

4217

  if (!pWins) {

×

4218

    code = terrno;

×

4219

    QUERY_CHECK_CODE(code, lino, _end);

×

4221

  code = doDeleteTimeWindows(pAggSup, pBlock, pWins);

×

4222

  QUERY_CHECK_CODE(code, lino, _end);

×

4224

  removeSessionResults(pAggSup, pMapUpdate, pWins);

×

4225

  code = copyDeleteWindowInfo(pWins, pMapDelete);

×

4226

  QUERY_CHECK_CODE(code, lino, _end);

×

4228

  if (needAdd) {

×

4229

    code = copyDeleteWindowInfo(pWins, pPkDelete);

×

4230

    QUERY_CHECK_CODE(code, lino, _end);

×

4232

  taosArrayDestroy(pWins);

×

4234

_end:

×

4235

  if (code != TSDB_CODE_SUCCESS) {

×

4236

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

4238

  return code;

×

4241

static int32_t doStreamSessionSemiAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

4242

  int32_t                        code = TSDB_CODE_SUCCESS;

×

4243

  int32_t                        lino = 0;

×

4244

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

4245

  SOptrBasicInfo*                pBInfo = &pInfo->binfo;

×

4246

  TSKEY                          maxTs = INT64_MIN;

×

4247

  SExprSupp*                     pSup = &pOperator->exprSupp;

×

4248

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

4249

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

4251

  qDebug("stask:%s  %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);

×

4252

  if (pOperator->status == OP_EXEC_DONE) {

×

4253

    (*ppRes) = NULL;

×

4254

    return code;

×

4258

    SSDataBlock* opRes = NULL;

×

4259

    code = buildSessionResult(pOperator, &opRes);

×

4260

    QUERY_CHECK_CODE(code, lino, _end);

×

4261

    if (opRes) {

×

4262

      (*ppRes) = opRes;

×

4263

      return code;

×

4266

    if (pInfo->clearState) {

×

4267

      clearFunctionContext(&pOperator->exprSupp);

×

4269

      clearStreamSessionOperator(pInfo);

×

4272

    if (pOperator->status == OP_RES_TO_RETURN) {

×

4273

      if (pInfo->reCkBlock) {

×

4274

        pInfo->reCkBlock = false;

×

4275

        printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

4276

        (*ppRes) = pInfo->pCheckpointRes;

×

4277

        return code;

×

4279

      clearFunctionContext(&pOperator->exprSupp);

×

4281

      clearStreamSessionOperator(pInfo);

×

4282

      setStreamOperatorCompleted(pOperator);

×

4283

      (*ppRes) = NULL;

×

4284

      return code;

×

4288

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

4289

  if (!pInfo->pUpdated) {

×

4290

    pInfo->pUpdated = taosArrayInit(16, sizeof(SResultWindowInfo));

×

4291

    QUERY_CHECK_NULL(pInfo->pUpdated, code, lino, _end, terrno);

×

4293

  if (!pInfo->pStUpdated) {

×

4294

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

4295

    pInfo->pStUpdated = tSimpleHashInit(64, hashFn);

×

4296

    QUERY_CHECK_NULL(pInfo->pStUpdated, code, lino, _end, terrno);

×

4298

  while (1) {

×

4299

    SSDataBlock* pBlock = NULL;

×

4300

    code = downstream->fpSet.getNextFn(downstream, &pBlock);

×

4301

    QUERY_CHECK_CODE(code, lino, _end);

×

4303

    if (pBlock == NULL) {

×

4304

      pOperator->status = OP_RES_TO_RETURN;

×

4305

      break;

×

4307

    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));

×

4308

    setStreamOperatorState(&pInfo->basic, pBlock->info.type);

×

4310

    if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||

×

4311

        pBlock->info.type == STREAM_CLEAR) {

×

4313

      code = deleteSessionWinState(pAggSup, pBlock, pInfo->pStUpdated, pInfo->pStDeleted, NULL, false);

×

4314

      QUERY_CHECK_CODE(code, lino, _end);

×

4315

      pInfo->clearState = true;

×

4316

      break;

×

4317

    } else if (pBlock->info.type == STREAM_GET_ALL) {

×

4318

      code = getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pStUpdated);

×

4319

      QUERY_CHECK_CODE(code, lino, _end);

×

4320

      continue;

×

4321

    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {

×

4322

      (*ppRes) = pBlock;

×

4323

      return code;

×

4324

    } else if (pBlock->info.type == STREAM_CHECKPOINT) {

×

4325

      pAggSup->stateStore.streamStateCommit(pAggSup->pState);

×

4326

      doStreamSessionSaveCheckpoint(pOperator);

×

4327

      continue;

×

4329

      if (pBlock->info.type != STREAM_NORMAL && pBlock->info.type != STREAM_INVALID) {

×

4330

        code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;

×

4331

        QUERY_CHECK_CODE(code, lino, _end);

×

4335

    if (pInfo->scalarSupp.pExprInfo != NULL) {

×

4336

      SExprSupp* pExprSup = &pInfo->scalarSupp;

×

4337

      code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);

×

4338

      QUERY_CHECK_CODE(code, lino, _end);

×

4341

    code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

4342

    QUERY_CHECK_CODE(code, lino, _end);

×

4343

    doStreamSessionAggImpl(pOperator, pBlock, pInfo->pStUpdated, NULL, false, false);

×

4344

    maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);

×

4347

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);

×

4348

  pBInfo->pRes->info.watermark = pInfo->twAggSup.maxTs;

×

4350

  code = copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc);

×

4351

  QUERY_CHECK_CODE(code, lino, _end);

×

4353

  removeSessionDeleteResults(pInfo->pStDeleted, pInfo->pUpdated);

×

4355

  if (pInfo->isHistoryOp) {

×

4356

    code = getMaxTsWins(pInfo->pUpdated, pInfo->historyWins);

×

4357

    QUERY_CHECK_CODE(code, lino, _end);

×

4360

  initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);

×

4361

  pInfo->pUpdated = NULL;

×

4362

  code = blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);

×

4363

  QUERY_CHECK_CODE(code, lino, _end);

×

4365

  SSDataBlock* opRes = NULL;

×

4366

  code = buildSessionResult(pOperator, &opRes);

×

4367

  QUERY_CHECK_CODE(code, lino, _end);

×

4368

  if (opRes) {

×

4369

    (*ppRes) = opRes;

×

4370

    return code;

×

4373

_end:

×

4374

  if (code != TSDB_CODE_SUCCESS) {

×

4375

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

4376

    pTaskInfo->code = code;

×

4377

    T_LONG_JMP(pTaskInfo->env, code);

×

4380

  clearFunctionContext(&pOperator->exprSupp);

×

4382

  clearStreamSessionOperator(pInfo);

×

4383

  setStreamOperatorCompleted(pOperator);

×

4384

  (*ppRes) = NULL;

×

4385

  return code;

×

4388

static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {

×

4389

  SSDataBlock* pRes = NULL;

×

4390

  int32_t      code = doStreamSessionSemiAggNext(pOperator, &pRes);

×

4391

  return pRes;

×

4394

int32_t createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,

×

4397

  QRY_PARAM_CHECK(pOptrInfo);

×

4399

  int32_t        code = TSDB_CODE_SUCCESS;

×

4400

  int32_t        lino = 0;

×

4401

  SOperatorInfo* pOperator = NULL;

×

4402

  code = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo, pHandle, &pOperator);

×

4403

  if (pOperator == NULL || code != 0) {

×

4404

    downstream = NULL;

×

4405

    QUERY_CHECK_CODE(code, lino, _error);

×

4408

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

4409

  pInfo->pOperator = pOperator;

×

4411

  if (pPhyNode->type != QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {

×

4412

    pOperator->fpSet =

×

4413

        createOperatorFpSet(optrDummyOpenFn, doStreamSessionSemiAggNext, NULL, destroyStreamSessionAggOperatorInfo,

×

4415

    setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionSemiReloadState);

×

4418

  if (numOfChild > 0) {

×

4419

    pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*));

×

4420

    QUERY_CHECK_NULL(pInfo->pChildren, code, lino, _error, terrno);

×

4421

    for (int32_t i = 0; i < numOfChild; i++) {

×

4422

      SOperatorInfo* pChildOp = NULL;

×

4423

      code = createStreamFinalSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo, 0, pHandle, &pChildOp);

×

4424

      if (pChildOp == NULL || code != 0) {

×

4425

        QUERY_CHECK_CODE(code, lino, _error);

×

4428

      SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info;

×

4429

      pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;

×

4430

      pTaskInfo->storageAPI.stateStore.streamStateSetNumber(pChInfo->streamAggSup.pState, i, pInfo->primaryTsIndex);

×

4431

      void* tmp = taosArrayPush(pInfo->pChildren, &pChildOp);

×

4432

      if (!tmp) {

×

4433

        code = terrno;

×

4434

        QUERY_CHECK_CODE(code, lino, _error);

×

4438

    void*   buff = NULL;

×

4439

    int32_t len = 0;

×

4441

        pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME,

×

4443

    if (res == TSDB_CODE_SUCCESS) {

×

4444

      code = doStreamSessionDecodeOpState(buff, len, pOperator, true, NULL);

×

4445

      taosMemoryFree(buff);

×

4446

      QUERY_CHECK_CODE(code, lino, _error);

×

4450

  if (!IS_FINAL_SESSION_OP(pOperator) || numOfChild == 0) {

×

4451

    pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;

×

4454

  *pOptrInfo = pOperator;

×

4455

  return code;

×

4457

_error:

×

4458

  if (pInfo != NULL) {

×

4459

    destroyStreamSessionAggOperatorInfo(pInfo);

×

4461

  if (pOperator != NULL) {

×

4462

    pOperator->info = NULL;

×

4463

    destroyOperator(pOperator);

×

4465

  pTaskInfo->code = code;

×

4466

  if (code != TSDB_CODE_SUCCESS) {

×

4467

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

4469

  return code;

×

4472

void destroyStreamStateOperatorInfo(void* param) {

×

4473

  if (param == NULL) {

×

4474

    return;

×

4476

  SStreamStateAggOperatorInfo* pInfo = (SStreamStateAggOperatorInfo*)param;

×

4477

  cleanupBasicInfo(&pInfo->binfo);

×

4478

  if (pInfo->pOperator) {

×

4479

    cleanupResultInfoInStream(pInfo->pOperator->pTaskInfo, pInfo->streamAggSup.pState, &pInfo->pOperator->exprSupp,

×

4481

    pInfo->pOperator = NULL;

×

4484

  destroyStreamBasicInfo(&pInfo->basic);

×

4485

  clearSessionGroupResInfo(&pInfo->groupResInfo);

×

4486

  taosArrayDestroyEx(pInfo->pUpdated, destroyResultWinInfo);

×

4487

  pInfo->pUpdated = NULL;

×

4488

  destroyStreamAggSupporter(&pInfo->streamAggSup);

×

4490

  cleanupExprSupp(&pInfo->scalarSupp);

×

4491

  if (pInfo->pChildren != NULL) {

×

4492

    int32_t size = taosArrayGetSize(pInfo->pChildren);

×

4493

    for (int32_t i = 0; i < size; i++) {

×

4494

      SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, i);

×

4495

      destroyOperator(pChild);

×

4497

    taosArrayDestroy(pInfo->pChildren);

×

4499

  colDataDestroy(&pInfo->twAggSup.timeWindowData);

×

4500

  blockDataDestroy(pInfo->pDelRes);

×

4501

  tSimpleHashCleanup(pInfo->pSeUpdated);

×

4502

  tSimpleHashCleanup(pInfo->pSeDeleted);

×

4503

  cleanupGroupResInfo(&pInfo->groupResInfo);

×

4505

  taosArrayDestroy(pInfo->historyWins);

×

4506

  blockDataDestroy(pInfo->pCheckpointRes);

×

4507

  tSimpleHashCleanup(pInfo->pPkDeleted);

×

4508

  destroyNonBlockAggSupptor(&pInfo->nbSup);

×

4510

  taosMemoryFreeClear(param);

×

4513

bool isTsInWindow(SStateWindowInfo* pWin, TSKEY ts) {

×

4514

  if (pWin->winInfo.sessionWin.win.skey <= ts && ts <= pWin->winInfo.sessionWin.win.ekey) {

×

4515

    return true;

×

4517

  return false;

×

4520

bool isEqualStateKey(SStateWindowInfo* pWin, char* pKeyData) {

×

4521

  return pKeyData && compareVal(pKeyData, pWin->pStateKey);

×

4524

bool compareStateKey(void* data, void* key) {

×

4525

  if (!data || !key) {

×

4526

    return true;

×

4528

  SStateKeys* stateKey = (SStateKeys*)key;

×

4529

  stateKey->pData = (char*)key + sizeof(SStateKeys);

×

4530

  return compareVal(data, stateKey);

×

4533

bool compareWinStateKey(SStateKeys* left, SStateKeys* right) {

×

4534

  if (!left || !right) {

×

4535

    return false;

×

4537

  return compareVal(left->pData, right);

×

4540

void getNextStateWin(const SStreamAggSupporter* pAggSup, SStateWindowInfo* pNextWin, bool asc) {

×

4541

  SStreamStateCur* pCur = NULL;

×

4543

  if (pAggSup == NULL || pNextWin == NULL) {

×

4544

    return;

×

4547

  if (asc)

×

4548

    pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pNextWin->winInfo.sessionWin);

×

4550

    pCur = pAggSup->stateStore.streamStateSessionSeekKeyPrev(pAggSup->pState, &pNextWin->winInfo.sessionWin);

×

4551

  int32_t nextSize = pAggSup->resultRowSize;

×

4552

  int32_t winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin,

×

4553

                                                                     (void**)&pNextWin->winInfo.pStatePos, &nextSize);

×

4554

  if (winCode != TSDB_CODE_SUCCESS) {

×

4555

    SET_SESSION_WIN_INVALID(pNextWin->winInfo);

×

4557

    pNextWin->pStateKey =

×

4558

        (SStateKeys*)((char*)pNextWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));

×

4559

    pNextWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);

×

4560

    pNextWin->pStateKey->type = pAggSup->stateKeyType;

×

4561

    pNextWin->pStateKey->pData = (char*)pNextWin->pStateKey + sizeof(SStateKeys);

×

4562

    pNextWin->pStateKey->isNull = false;

×

4563

    pNextWin->winInfo.isOutput = true;

×

4565

  pAggSup->stateStore.streamStateFreeCur(pCur);

×

4568

int32_t getStateWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SStateWindowInfo* pCurWin,

×

4570

  int32_t code = TSDB_CODE_SUCCESS;

×

4571

  int32_t lino = 0;

×

4572

  int32_t size = pAggSup->resultRowSize;

×

4573

  pCurWin->winInfo.sessionWin.groupId = pKey->groupId;

×

4574

  pCurWin->winInfo.sessionWin.win.skey = pKey->win.skey;

×

4575

  pCurWin->winInfo.sessionWin.win.ekey = pKey->win.ekey;

×

4576

  code = getSessionWindowInfoByKey(pAggSup, pKey, &pCurWin->winInfo);

×

4577

  QUERY_CHECK_CODE(code, lino, _end);

×

4578

  QUERY_CHECK_CONDITION((IS_VALID_SESSION_WIN(pCurWin->winInfo)), code, lino, _end,

×

4581

  pCurWin->pStateKey =

×

4582

      (SStateKeys*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));

×

4583

  pCurWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);

×

4584

  pCurWin->pStateKey->type = pAggSup->stateKeyType;

×

4585

  pCurWin->pStateKey->pData = (char*)pCurWin->pStateKey + sizeof(SStateKeys);

×

4586

  pCurWin->pStateKey->isNull = false;

×

4587

  pCurWin->winInfo.isOutput = true;

×

4588

  if (pCurWin->winInfo.pStatePos->needFree) {

×

4589

    pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->winInfo.sessionWin);

×

4592

  qDebug("===stream===get state cur win buff. skey:%" PRId64 ", endkey:%" PRId64, pCurWin->winInfo.sessionWin.win.skey,

×

4595

  pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin;

×

4596

  getNextStateWin(pAggSup, pNextWin, true);

×

4598

_end:

×

4599

  qDebug("===stream===get state next win buff. skey:%" PRId64 ", endkey:%" PRId64,

×

4601

  if (code != TSDB_CODE_SUCCESS) {

×

4602

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

4604

  return code;

×

4607

int32_t setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData,

×

4609

  int32_t size = pAggSup->resultRowSize;

×

4610

  pCurWin->winInfo.sessionWin.groupId = groupId;

×

4611

  pCurWin->winInfo.sessionWin.win.skey = ts;

×

4612

  pCurWin->winInfo.sessionWin.win.ekey = ts;

×

4613

  int32_t code = TSDB_CODE_SUCCESS;

×

4614

  int32_t lino = 0;

×

4615

  int32_t winCode = TSDB_CODE_SUCCESS;

×

4616

  code = pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, pKeyData,

×

4618

                                                           (void**)&pCurWin->winInfo.pStatePos, &size, &winCode);

×

4619

  QUERY_CHECK_CODE(code, lino, _end);

×

4621

  pCurWin->pStateKey =

×

4622

      (SStateKeys*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));

×

4623

  pCurWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);

×

4624

  pCurWin->pStateKey->type = pAggSup->stateKeyType;

×

4625

  pCurWin->pStateKey->pData = (char*)pCurWin->pStateKey + sizeof(SStateKeys);

×

4626

  pCurWin->pStateKey->isNull = false;

×

4628

  if (winCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->winInfo.sessionWin.win)) {

×

4629

    winCode = TSDB_CODE_FAILED;

×

4630

    clearOutputBuf(pAggSup->pState, pCurWin->winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);

×

4631

    pCurWin->pStateKey =

×

4632

        (SStateKeys*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));

×

4633

    pCurWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);

×

4634

    pCurWin->pStateKey->type = pAggSup->stateKeyType;

×

4635

    pCurWin->pStateKey->pData = (char*)pCurWin->pStateKey + sizeof(SStateKeys);

×

4636

    pCurWin->pStateKey->isNull = false;

×

4637

    pCurWin->winInfo.sessionWin.groupId = groupId;

×

4638

    pCurWin->winInfo.sessionWin.win.skey = ts;

×

4639

    pCurWin->winInfo.sessionWin.win.ekey = ts;

×

4640

    qDebug("===stream===reset state win key. skey:%" PRId64 ", endkey:%" PRId64, pCurWin->winInfo.sessionWin.win.skey,

×

4644

  if (winCode == TSDB_CODE_SUCCESS) {

×

4645

    pCurWin->winInfo.isOutput = true;

×

4646

    if (pCurWin->winInfo.pStatePos->needFree) {

×

4647

      pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->winInfo.sessionWin);

×

4649

  } else if (pKeyData) {

×

4650

    if (IS_VAR_DATA_TYPE(pAggSup->stateKeyType)) {

×

4651

      varDataCopy(pCurWin->pStateKey->pData, pKeyData);

×

4653

      memcpy(pCurWin->pStateKey->pData, pKeyData, pCurWin->pStateKey->bytes);

×

4657

  *pWinCode = winCode;

×

4659

  qDebug("===stream===set state cur win buff. skey:%" PRId64 ", endkey:%" PRId64, pCurWin->winInfo.sessionWin.win.skey,

×

4662

  pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin;

×

4663

  getNextStateWin(pAggSup, pNextWin, true);

×

4664

  qDebug("===stream===set state next win buff. skey:%" PRId64 ", endkey:%" PRId64,

×

4666

_end:

×

4667

  if (code != TSDB_CODE_SUCCESS) {

×

4668

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

4670

  return code;

×

4673

int32_t updateStateWindowInfo(SStreamAggSupporter* pAggSup, SStateWindowInfo* pWinInfo, SStateWindowInfo* pNextWin,

×

4677

  int32_t code = TSDB_CODE_SUCCESS;

×

4678

  int32_t lino = 0;

×

4679

  *allEqual = true;

×

4680

  for (int32_t i = start; i < rows; ++i) {

×

4681

    char* pKeyData = colDataGetData(pKeyCol, i);

×

4682

    if (!isTsInWindow(pWinInfo, pTs[i])) {

×

4683

      if (isEqualStateKey(pWinInfo, pKeyData)) {

×

4684

        if (IS_VALID_SESSION_WIN(pNextWin->winInfo)) {

×

4686

          if (pTs[i] >= pNextWin->winInfo.sessionWin.win.skey) {

×

4687

            (*pWinRows) = i - start;

×

4688

            goto _end;

×

4692

        (*pWinRows) = i - start;

×

4693

        goto _end;

×

4697

    if (pWinInfo->winInfo.sessionWin.win.skey > pTs[i]) {

×

4698

      if (pSeDeleted && pWinInfo->winInfo.isOutput) {

×

4699

        code = saveDeleteRes(pSeDeleted, pWinInfo->winInfo.sessionWin);

×

4700

        QUERY_CHECK_CODE(code, lino, _end);

×

4702

      removeSessionResult(pAggSup, pSeUpdated, pResultRows, &pWinInfo->winInfo.sessionWin);

×

4703

      pWinInfo->winInfo.sessionWin.win.skey = pTs[i];

×

4705

    pWinInfo->winInfo.sessionWin.win.ekey = TMAX(pWinInfo->winInfo.sessionWin.win.ekey, pTs[i]);

×

4706

    memcpy(pWinInfo->winInfo.pStatePos->pKey, &pWinInfo->winInfo.sessionWin, sizeof(SSessionKey));

×

4707

    if (!isEqualStateKey(pWinInfo, pKeyData)) {

×

4708

      *allEqual = false;

×

4711

  (*pWinRows) = rows - start;

×

4713

_end:

×

4714

  if (code != TSDB_CODE_SUCCESS) {

×

4715

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

4717

  return code;

×

4720

static bool isWinResult(SSessionKey* pKey, SSHashObj* pSeUpdate, SSHashObj* pResults) {

×

4721

  SSessionKey checkKey = {0};

×

4722

  getSessionHashKey(pKey, &checkKey);

×

4723

  if (tSimpleHashGet(pSeUpdate, &checkKey, sizeof(SSessionKey)) != NULL) {

×

4724

    return true;

×

4727

  if (tSimpleHashGet(pResults, &checkKey, sizeof(SSessionKey)) != NULL) {

×

4728

    return true;

×

4730

  return false;

×

4733

static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pSeUpdated,

×

4735

  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

×

4736

  SStorageAPI*   pAPI = &pOperator->pTaskInfo->storageAPI;

×

4738

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

4739

  int32_t                      numOfOutput = pOperator->exprSupp.numOfExprs;

×

4740

  uint64_t                     groupId = pSDataBlock->info.id.groupId;

×

4741

  int32_t                      code = TSDB_CODE_SUCCESS;

×

4742

  int32_t                      lino = 0;

×

4743

  TSKEY*                       tsCols = NULL;

×

4744

  SResultRow*                  pResult = NULL;

×

4745

  int32_t                      winRows = 0;

×

4746

  SStreamAggSupporter*         pAggSup = &pInfo->streamAggSup;

×

4747

  SStreamNotifyEventSupp*      pNotifySup = &pInfo->basic.notifyEventSup;

×

4748

  STaskNotifyEventStat*        pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;

×

4750

  pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);

×

4751

  pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;

×

4752

  if (pAggSup->winRange.ekey <= 0) {

×

4753

    pAggSup->winRange.ekey = INT64_MAX;

×

4756

  if (pSDataBlock->pDataBlock != NULL) {

×

4757

    SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);

×

4758

    if (!pColDataInfo) {

×

4759

      code = TSDB_CODE_FAILED;

×

4760

      QUERY_CHECK_CODE(code, lino, _end);

×

4762

    tsCols = (int64_t*)pColDataInfo->pData;

×

4764

    return;

×

4767

  int32_t rows = pSDataBlock->info.rows;

×

4768

  code = blockDataEnsureCapacity(pAggSup->pScanBlock, rows);

×

4769

  QUERY_CHECK_CODE(code, lino, _end);

×

4771

  SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);

×

4772

  for (int32_t i = 0; i < rows; i += winRows) {

×

4773

    if (pInfo->ignoreExpiredData && checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo,

×

4774

                                                     &pInfo->twAggSup, pSDataBlock->info.id.uid, tsCols[i], NULL, 0) ||

×

4775

        colDataIsNull_s(pKeyColInfo, i)) {

×

4776

      i++;

×

4777

      continue;

×

4779

    char*            pKeyData = colDataGetData(pKeyColInfo, i);

×

4780

    int32_t          winIndex = 0;

×

4781

    bool             allEqual = true;

×

4782

    SStateWindowInfo curWin = {0};

×

4783

    SStateWindowInfo nextWin = {0};

×

4784

    int32_t          winCode = TSDB_CODE_SUCCESS;

×

4785

    code = setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin, &winCode);

×

4786

    QUERY_CHECK_CODE(code, lino, _end);

×

4788

    if (winCode != TSDB_CODE_SUCCESS && pTaskInfo->streamInfo.eventTypes) {

×

4789

      SStateWindowInfo prevWin = {.winInfo.sessionWin = curWin.winInfo.sessionWin};

×

4790

      getNextStateWin(pAggSup, &prevWin, false);

×

4791

      qDebug("===stream===get state prev win buff. skey:%" PRId64 ", endkey:%" PRId64,

×

4793

      releaseOutputBuf(pAggSup->pState, prevWin.winInfo.pStatePos, &pAPI->stateStore);

×

4796

      if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE) &&

×

4797

          IS_VALID_SESSION_WIN(prevWin.winInfo)) {

×

4798

        code = addStateAggNotifyEvent(SNOTIFY_EVENT_WINDOW_CLOSE, &prevWin.winInfo.sessionWin, prevWin.pStateKey,

×

4799

                                      curWin.pStateKey, true, pNotifySup, pNotifyEventStat);

×

4800

        QUERY_CHECK_CODE(code, lino, _end);

×

4802

      if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {

×

4803

        code = addStateAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &curWin.winInfo.sessionWin, curWin.pStateKey,

×

4804

                                      prevWin.pStateKey, false, pNotifySup, pNotifyEventStat);

×

4805

        QUERY_CHECK_CODE(code, lino, _end);

×

4809

    if (isWinResult(&nextWin.winInfo.sessionWin, pSeUpdated, pAggSup->pResultRows) == false) {

×

4810

      releaseOutputBuf(pAggSup->pState, nextWin.winInfo.pStatePos, &pAPI->stateStore);

×

4813

    setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo);

×

4814

    code = updateStateWindowInfo(pAggSup, &curWin, &nextWin, tsCols, groupId, pKeyColInfo, rows, i, &allEqual,

×

4816

    QUERY_CHECK_CODE(code, lino, _end);

×

4818

    if (!allEqual) {

×

4819

      uint64_t uid = 0;

×

4820

      code = appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey,

×

4822

      QUERY_CHECK_CODE(code, lino, _end);

×

4823

      int32_t tmpRes = tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey));

×

4824

      qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

4826

      doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin);

×

4827

      releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore);

×

4828

      continue;

×

4831

    code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput,

×

4833

    QUERY_CHECK_CODE(code, lino, _end);

×

4835

    code = saveSessionOutputBuf(pAggSup, &curWin.winInfo);

×

4836

    QUERY_CHECK_CODE(code, lino, _end);

×

4838

    if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_STATE_OP(pOperator)) {

×

4839

      code = saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin);

×

4840

      QUERY_CHECK_CODE(code, lino, _end);

×

4843

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {

×

4844

      code = saveResult(curWin.winInfo, pSeUpdated);

×

4845

      QUERY_CHECK_CODE(code, lino, _end);

×

4848

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

4849

      curWin.winInfo.pStatePos->beUpdated = true;

×

4850

      SSessionKey key = {0};

×

4851

      getSessionHashKey(&curWin.winInfo.sessionWin, &key);

×

4853

          tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo));

×

4854

      QUERY_CHECK_CODE(code, lino, _end);

×

4859

    if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE)) {

×

4860

      code = addStateAggNotifyEvent(SNOTIFY_EVENT_WINDOW_CLOSE, &curWin.winInfo.sessionWin, curWin.pStateKey,

×

4861

                                    nextWin.pStateKey, false, pNotifySup, pNotifyEventStat);

×

4862

      QUERY_CHECK_CODE(code, lino, _end);

×

4866

_end:

×

4867

  if (code != TSDB_CODE_SUCCESS) {

×

4868

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

4872

int32_t doStreamStateEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOperator, bool isParent) {

×

4873

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

4874

  if (!pInfo) {

×

4875

    return 0;

×

4878

  void* pData = (buf == NULL) ? NULL : *buf;

×

4881

  int32_t tlen = 0;

×

4882

  int32_t mapSize = tSimpleHashGetSize(pInfo->streamAggSup.pResultRows);

×

4883

  tlen += taosEncodeFixedI32(buf, mapSize);

×

4884

  void*   pIte = NULL;

×

4885

  size_t  keyLen = 0;

×

4886

  int32_t iter = 0;

×

4887

  while ((pIte = tSimpleHashIterate(pInfo->streamAggSup.pResultRows, pIte, &iter)) != NULL) {

×

4888

    void* key = tSimpleHashGetKey(pIte, &keyLen);

×

4889

    tlen += encodeSSessionKey(buf, key);

×

4890

    tlen += encodeSResultWindowInfo(buf, pIte, pInfo->streamAggSup.resultRowSize);

×

4894

  tlen += encodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);

×

4897

  int32_t size = taosArrayGetSize(pInfo->pChildren);

×

4898

  tlen += taosEncodeFixedI32(buf, size);

×

4899

  for (int32_t i = 0; i < size; i++) {

×

4900

    SOperatorInfo* pChOp = taosArrayGetP(pInfo->pChildren, i);

×

4901

    tlen += doStreamStateEncodeOpState(buf, 0, pChOp, false);

×

4905

  tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);

×

4908

  tlen += encodeStreamBasicInfo(buf, &pInfo->basic);

×

4911

  if (isParent) {

×

4912

    if (buf) {

×

4913

      uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));

×

4914

      tlen += taosEncodeFixedU32(buf, cksum);

×

4916

      tlen += sizeof(uint32_t);

×

4920

  return tlen;

×

4923

int32_t doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator, bool isParent, void** ppBuf) {

×

4924

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

4925

  int32_t                      code = TSDB_CODE_SUCCESS;

×

4926

  int32_t                      lino = 0;

×

4927

  SStreamAggSupporter*         pAggSup = &pInfo->streamAggSup;

×

4928

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

4929

  void*                        pDataEnd = POINTER_SHIFT(buf, len);

×

4930

  if (!pInfo) {

×

4931

    code = TSDB_CODE_FAILED;

×

4932

    QUERY_CHECK_CODE(code, lino, _end);

×

4936

  if (isParent) {

×

4937

    int32_t dataLen = len - sizeof(uint32_t);

×

4938

    void*   pCksum = POINTER_SHIFT(buf, dataLen);

×

4939

    if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) {

×

4940

      qError("stream state_window state is invalid");

×

4941

      code = TSDB_CODE_FAILED;

×

4942

      QUERY_CHECK_CODE(code, lino, _end);

×

4944

    pDataEnd = pCksum;

×

4948

  int32_t mapSize = 0;

×

4949

  buf = taosDecodeFixedI32(buf, &mapSize);

×

4950

  for (int32_t i = 0; i < mapSize; i++) {

×

4951

    SResultWindowInfo winfo = {0};

×

4952

    buf = decodeSSessionKey(buf, &winfo.sessionWin);

×

4953

    int32_t winCode = TSDB_CODE_SUCCESS;

×

4954

    code = pAggSup->stateStore.streamStateStateAddIfNotExist(

×

4957

    QUERY_CHECK_CODE(code, lino, _end);

×

4959

    buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize);

×

4960

    code = tSimpleHashPut(pInfo->streamAggSup.pResultRows, &winfo.sessionWin, sizeof(SSessionKey), &winfo,

×

4962

    QUERY_CHECK_CODE(code, lino, _end);

×

4966

  buf = decodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);

×

4969

  int32_t size = 0;

×

4970

  buf = taosDecodeFixedI32(buf, &size);

×

4971

  for (int32_t i = 0; i < size; i++) {

×

4972

    SOperatorInfo* pChOp = taosArrayGetP(pInfo->pChildren, i);

×

4973

    code = doStreamStateDecodeOpState(buf, 0, pChOp, false, &buf);

×

4974

    QUERY_CHECK_CODE(code, lino, _end);

×

4978

  buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);

×

4980

  if (ppBuf) {

×

4981

    (*ppBuf) = buf;

×

4985

  if (buf < pDataEnd) {

×

4986

    code = decodeStreamBasicInfo(&buf, &pInfo->basic);

×

4987

    QUERY_CHECK_CODE(code, lino, _end);

×

4990

_end:

×

4991

  if (code != TSDB_CODE_SUCCESS) {

×

4992

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

4994

  return code;

×

4997

void doStreamStateSaveCheckpoint(SOperatorInfo* pOperator) {

×

4998

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

4999

  if (needSaveStreamOperatorInfo(&pInfo->basic)) {

×

5000

    int32_t len = doStreamStateEncodeOpState(NULL, 0, pOperator, true);

×

5001

    void*   buf = taosMemoryCalloc(1, len);

×

5002

    if (!buf) {

×

5003

      qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY));

×

5004

      return;

×

5006

    void* pBuf = buf;

×

5007

    len = doStreamStateEncodeOpState(&pBuf, len, pOperator, true);

×

5008

    pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_CHECKPOINT_NAME,

×

5010

    taosMemoryFree(buf);

×

5011

    saveStreamOperatorStateComplete(&pInfo->basic);

×

5015

static int32_t buildStateResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

5016

  int32_t                      code = TSDB_CODE_SUCCESS;

×

5017

  int32_t                      lino = 0;

×

5018

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

5019

  SOptrBasicInfo*              pBInfo = &pInfo->binfo;

×

5020

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

5021

  SStreamNotifyEventSupp*      pNotifySup = &pInfo->basic.notifyEventSup;

×

5022

  STaskNotifyEventStat*        pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;

×

5023

  bool                         addNotifyEvent = false;

×

5024

  addNotifyEvent = BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);

×

5025

  doBuildDeleteDataBlock(pOperator, pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator, &pInfo->groupResInfo);

×

5026

  if (pInfo->pDelRes->info.rows > 0) {

×

5027

    printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5028

    if (addNotifyEvent) {

×

5029

      code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);

×

5030

      QUERY_CHECK_CODE(code, lino, _end);

×

5032

    (*ppRes) = pInfo->pDelRes;

×

5033

    return code;

×

5036

  doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes,

×

5038

  if (pBInfo->pRes->info.rows > 0) {

×

5039

    printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5040

    if (addNotifyEvent) {

×

5041

      code = addAggResultNotifyEvent(pBInfo->pRes, pNotifySup->pSessionKeys, pTaskInfo->streamInfo.notifyResultSchema,

×

5043

      QUERY_CHECK_CODE(code, lino, _end);

×

5045

    (*ppRes) = pBInfo->pRes;

×

5046

    return code;

×

5049

  code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);

×

5050

  QUERY_CHECK_CODE(code, lino, _end);

×

5051

  if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {

×

5052

    printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5053

    (*ppRes) = pNotifySup->pEventBlock;

×

5054

    return code;

×

5057

  code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);

×

5058

  QUERY_CHECK_CODE(code, lino, _end);

×

5060

_end:

×

5061

  if (code != TSDB_CODE_SUCCESS) {

×

5062

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

5064

  (*ppRes) = NULL;

×

5065

  return code;

×

5068

static int32_t doStreamStateAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

5069

  if (pOperator->status == OP_EXEC_DONE) {

×

5070

    (*ppRes) = NULL;

×

5071

    return TSDB_CODE_SUCCESS;

×

5074

  int32_t                      code = TSDB_CODE_SUCCESS;

×

5075

  int32_t                      lino = 0;

×

5076

  SExprSupp*                   pSup = &pOperator->exprSupp;

×

5077

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

5078

  SOptrBasicInfo*              pBInfo = &pInfo->binfo;

×

5079

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

5080

  qDebug("===stream=== stream state agg");

×

5081

  if (pOperator->status == OP_RES_TO_RETURN) {

×

5082

    SSDataBlock* resBlock = NULL;

×

5083

    code = buildStateResult(pOperator, &resBlock);

×

5084

    QUERY_CHECK_CODE(code, lino, _end);

×

5085

    if (resBlock != NULL) {

×

5086

      (*ppRes) = resBlock;

×

5087

      return code;

×

5090

    if (pInfo->recvGetAll) {

×

5091

      pInfo->recvGetAll = false;

×

5092

      resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows);

×

5095

    if (pInfo->reCkBlock) {

×

5096

      pInfo->reCkBlock = false;

×

5097

      printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5098

      (*ppRes) = pInfo->pCheckpointRes;

×

5099

      return code;

×

5102

    setStreamOperatorCompleted(pOperator);

×

5103

    (*ppRes) = NULL;

×

5104

    return code;

×

5107

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

5108

  if (!pInfo->pUpdated) {

×

5109

    pInfo->pUpdated = taosArrayInit(16, sizeof(SResultWindowInfo));

×

5110

    QUERY_CHECK_NULL(pInfo->pUpdated, code, lino, _end, terrno);

×

5112

  if (!pInfo->pSeUpdated) {

×

5113

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

5114

    pInfo->pSeUpdated = tSimpleHashInit(64, hashFn);

×

5115

    QUERY_CHECK_NULL(pInfo->pSeUpdated, code, lino, _end, terrno);

×

5117

  while (1) {

×

5118

    SSDataBlock* pBlock = NULL;

×

5119

    code = downstream->fpSet.getNextFn(downstream, &pBlock);

×

5120

    QUERY_CHECK_CODE(code, lino, _end);

×

5122

    if (pBlock == NULL) {

×

5123

      break;

×

5125

    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));

×

5126

    setStreamOperatorState(&pInfo->basic, pBlock->info.type);

×

5128

    if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||

×

5129

        pBlock->info.type == STREAM_CLEAR) {

×

5130

      bool add = pInfo->destHasPrimaryKey && IS_NORMAL_STATE_OP(pOperator);

×

5131

      code = deleteSessionWinState(&pInfo->streamAggSup, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted,

×

5133

      QUERY_CHECK_CODE(code, lino, _end);

×

5134

      continue;

×

5135

    } else if (pBlock->info.type == STREAM_GET_ALL) {

×

5136

      pInfo->recvGetAll = true;

×

5137

      code = getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pSeUpdated);

×

5138

      QUERY_CHECK_CODE(code, lino, _end);

×

5139

      continue;

×

5140

    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {

×

5141

      (*ppRes) = pBlock;

×

5142

      return code;

×

5143

    } else if (pBlock->info.type == STREAM_CHECKPOINT) {

×

5144

      pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState);

×

5145

      doStreamStateSaveCheckpoint(pOperator);

×

5146

      code = copyDataBlock(pInfo->pCheckpointRes, pBlock);

×

5147

      QUERY_CHECK_CODE(code, lino, _end);

×

5149

      continue;

×

5151

      if (pBlock->info.type != STREAM_NORMAL && pBlock->info.type != STREAM_INVALID) {

×

5152

        code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;

×

5153

        QUERY_CHECK_CODE(code, lino, _end);

×

5157

    if (pInfo->scalarSupp.pExprInfo != NULL) {

×

5158

      SExprSupp* pExprSup = &pInfo->scalarSupp;

×

5159

      code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);

×

5160

      QUERY_CHECK_CODE(code, lino, _end);

×

5163

    code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

5164

    QUERY_CHECK_CODE(code, lino, _end);

×

5165

    doStreamStateAggImpl(pOperator, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted);

×

5166

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);

×

5169

  pOperator->status = OP_RES_TO_RETURN;

×

5171

  code = closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pInfo->pSeUpdated);

×

5172

  QUERY_CHECK_CODE(code, lino, _end);

×

5174

  code = copyUpdateResult(&pInfo->pSeUpdated, pInfo->pUpdated, sessionKeyCompareAsc);

×

5175

  QUERY_CHECK_CODE(code, lino, _end);

×

5177

  removeSessionDeleteResults(pInfo->pSeDeleted, pInfo->pUpdated);

×

5179

  if (pInfo->isHistoryOp) {

×

5180

    code = getMaxTsWins(pInfo->pUpdated, pInfo->historyWins);

×

5181

    QUERY_CHECK_CODE(code, lino, _end);

×

5183

  if (pInfo->destHasPrimaryKey && IS_NORMAL_STATE_OP(pOperator)) {

×

5184

    code = copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pSeDeleted);

×

5185

    QUERY_CHECK_CODE(code, lino, _end);

×

5188

  initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);

×

5189

  pInfo->pUpdated = NULL;

×

5190

  code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);

×

5191

  QUERY_CHECK_CODE(code, lino, _end);

×

5193

  SSDataBlock* resBlock = NULL;

×

5194

  code = buildStateResult(pOperator, &resBlock);

×

5195

  QUERY_CHECK_CODE(code, lino, _end);

×

5196

  if (resBlock != NULL) {

×

5197

    (*ppRes) = resBlock;

×

5198

    return code;

×

5201

_end:

×

5202

  if (code != TSDB_CODE_SUCCESS) {

×

5203

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

5204

    pTaskInfo->code = code;

×

5205

    T_LONG_JMP(pTaskInfo->env, code);

×

5207

  setStreamOperatorCompleted(pOperator);

×

5208

  (*ppRes) = NULL;

×

5209

  return code;

×

5212

static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {

×

5213

  SSDataBlock* pRes = NULL;

×

5214

  int32_t      code = doStreamStateAggNext(pOperator, &pRes);

×

5215

  return pRes;

×

5218

void streamStateReleaseState(SOperatorInfo* pOperator) {

×

5219

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

5220

  int32_t                      winSize = taosArrayGetSize(pInfo->historyWins) * sizeof(SSessionKey);

×

5221

  int32_t                      resSize = winSize + sizeof(TSKEY);

×

5222

  char*                        pBuff = taosMemoryCalloc(1, resSize);

×

5223

  if (!pBuff) {

×

5224

    return;

×

5226

  memcpy(pBuff, pInfo->historyWins->pData, winSize);

×

5227

  memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY));

×

5228

  qDebug("===stream=== relase state. save result count:%d", (int32_t)taosArrayGetSize(pInfo->historyWins));

×

5229

  pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_STATE_NAME,

×

5231

  pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState);

×

5232

  taosMemoryFreeClear(pBuff);

×

5234

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

5235

  if (downstream->fpSet.releaseStreamStateFn) {

×

5236

    downstream->fpSet.releaseStreamStateFn(downstream);

×

5240

int32_t compactStateWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin,

×

5242

  SExprSupp*                   pSup = &pOperator->exprSupp;

×

5243

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

5244

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

5245

  return compactTimeWindow(pSup, &pInfo->streamAggSup, &pInfo->twAggSup, pTaskInfo, pCurWin, pNextWin, pStUpdated,

×

5249

void streamStateReloadState(SOperatorInfo* pOperator) {

×

5250

  int32_t                      code = TSDB_CODE_SUCCESS;

×

5251

  int32_t                      lino = 0;

×

5252

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

5253

  SStreamAggSupporter*         pAggSup = &pInfo->streamAggSup;

×

5254

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

5255

  resetWinRange(&pAggSup->winRange);

×

5257

  SSessionKey seKey = {.win.skey = INT64_MIN, .win.ekey = INT64_MIN, .groupId = 0};

×

5258

  int32_t     size = 0;

×

5259

  void*       pBuf = NULL;

×

5260

  code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_STATE_OP_STATE_NAME,

×

5262

  QUERY_CHECK_CODE(code, lino, _end);

×

5264

  int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);

×

5265

  qDebug("===stream=== reload state. get result count:%d", num);

×

5266

  SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;

×

5268

  TSKEY ts = *(TSKEY*)((char*)pBuf + size - sizeof(TSKEY));

×

5269

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);

×

5270

  pAggSup->stateStore.streamStateReloadInfo(pAggSup->pState, ts);

×

5272

  if (!pInfo->pSeUpdated && num > 0) {

×

5273

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

5274

    pInfo->pSeUpdated = tSimpleHashInit(64, hashFn);

×

5275

    QUERY_CHECK_NULL(pInfo->pSeUpdated, code, lino, _end, terrno);

×

5277

  if (!pInfo->pSeDeleted && num > 0) {

×

5278

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

5279

    pInfo->pSeDeleted = tSimpleHashInit(64, hashFn);

×

5280

    QUERY_CHECK_NULL(pInfo->pSeDeleted, code, lino, _end, terrno);

×

5282

  for (int32_t i = 0; i < num; i++) {

×

5283

    SStateWindowInfo curInfo = {0};

×

5284

    SStateWindowInfo nextInfo = {0};

×

5285

    qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey,

×

5287

    code = getStateWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo, &nextInfo);

×

5288

    QUERY_CHECK_CODE(code, lino, _end);

×

5290

    bool cpRes = compareWinStateKey(curInfo.pStateKey, nextInfo.pStateKey);

×

5291

    qDebug("===stream=== reload state. next window info %" PRId64 ", %" PRIu64 ", compare:%d",

×

5293

    if (cpRes) {

×

5294

      code = compactStateWindow(pOperator, &curInfo.winInfo, &nextInfo.winInfo, pInfo->pSeUpdated, pInfo->pSeDeleted);

×

5295

      qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey,

×

5297

      QUERY_CHECK_CODE(code, lino, _end);

×

5299

      if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {

×

5300

        code = saveResult(curInfo.winInfo, pInfo->pSeUpdated);

×

5301

        QUERY_CHECK_CODE(code, lino, _end);

×

5302

      } else if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

5303

        if (!isCloseWindow(&curInfo.winInfo.sessionWin.win, &pInfo->twAggSup)) {

×

5304

          code = saveDeleteRes(pInfo->pSeDeleted, curInfo.winInfo.sessionWin);

×

5305

          QUERY_CHECK_CODE(code, lino, _end);

×

5307

        SSessionKey key = {0};

×

5308

        getSessionHashKey(&curInfo.winInfo.sessionWin, &key);

×

5309

        code = tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curInfo.winInfo,

×

5311

        QUERY_CHECK_CODE(code, lino, _end);

×

5313

    } else if (IS_VALID_SESSION_WIN(nextInfo.winInfo)) {

×

5314

      releaseOutputBuf(pAggSup->pState, nextInfo.winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);

×

5317

    if (IS_VALID_SESSION_WIN(curInfo.winInfo)) {

×

5318

      code = saveSessionOutputBuf(pAggSup, &curInfo.winInfo);

×

5319

      QUERY_CHECK_CODE(code, lino, _end);

×

5322

  taosMemoryFreeClear(pBuf);

×

5324

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

5325

  if (downstream->fpSet.reloadStreamStateFn) {

×

5326

    downstream->fpSet.reloadStreamStateFn(downstream);

×

5328

  reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup);

×

5330

_end:

×

5331

  taosMemoryFreeClear(pBuf);

×

5332

  if (code != TSDB_CODE_SUCCESS) {

×

5333

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

5335

×

5337

int32_t createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,

×

5339

  QRY_PARAM_CHECK(pOptrInfo);

×

5340

  int32_t code = 0;

×

5341

  int32_t lino = 0;

×

5343

  SStreamStateWinodwPhysiNode* pStateNode = (SStreamStateWinodwPhysiNode*)pPhyNode;

×

5344

  int32_t                      tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;

×

5345

  SColumnNode*                 pColNode = (SColumnNode*)(pStateNode->pStateKey);

×

5346

  SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo));

×

5347

  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));

×

5348

  if (pInfo == NULL || pOperator == NULL) {

×

5349

    code = terrno;

×

5350

    QUERY_CHECK_CODE(code, lino, _error);

×

5353

  pInfo->stateCol = extractColumnFromColumnNode(pColNode);

×

5354

  initResultSizeInfo(&pOperator->resultInfo, 4096);

×

5355

  if (pStateNode->window.pExprs != NULL) {

×

5356

    int32_t    numOfScalar = 0;

×

5357

    SExprInfo* pScalarExprInfo = NULL;

×

5358

    code = createExprInfo(pStateNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalar);

×

5359

    QUERY_CHECK_CODE(code, lino, _error);

×

5361

    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);

×

5362

    QUERY_CHECK_CODE(code, lino, _error);

×

5365

  pInfo->twAggSup = (STimeWindowAggSupp){

×

5366

      .waterMark = pStateNode->window.watermark,

×

5367

      .calTrigger = pStateNode->window.triggerType,

×

5370

      .deleteMark = getDeleteMark(&pStateNode->window, 0),

×

5373

  code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

×

5374

  QUERY_CHECK_CODE(code, lino, _error);

×

5376

  SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);

×

5377

  QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);

×

5378

  pInfo->binfo.pRes = pResBlock;

×

5380

  SExprSupp* pExpSup = &pOperator->exprSupp;

×

5381

  int32_t    numOfCols = 0;

×

5382

  SExprInfo* pExprInfo = NULL;

×

5383

  code = createExprInfo(pStateNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);

×

5384

  QUERY_CHECK_CODE(code, lino, _error);

×

5386

  code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore);

×

5387

  if (code != TSDB_CODE_SUCCESS) {

×

5388

    goto _error;

×

5390

  int32_t keySize = sizeof(SStateKeys) + pColNode->node.resType.bytes;

×

5391

  int16_t type = pColNode->node.resType.type;

×

5392

  pInfo->primaryTsIndex = tsSlotId;

×

5394

      initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState, keySize, type,

×

5395

                             &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup, GET_TASKID(pTaskInfo),

×

5397

  QUERY_CHECK_CODE(code, lino, _error);

×

5399

  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

5400

  pInfo->pSeDeleted = tSimpleHashInit(64, hashFn);

×

5401

  QUERY_CHECK_NULL(pInfo->pSeDeleted, code, lino, _error, terrno);

×

5402

  pInfo->pDelIterator = NULL;

×

5404

  code = createSpecialDataBlock(STREAM_DELETE_RESULT, &pInfo->pDelRes);

×

5405

  QUERY_CHECK_CODE(code, lino, _error);

×

5407

  pInfo->pChildren = NULL;

×

5408

  pInfo->ignoreExpiredData = pStateNode->window.igExpired;

×

5409

  pInfo->ignoreExpiredDataSaved = false;

×

5410

  pInfo->pUpdated = NULL;

×

5411

  pInfo->pSeUpdated = NULL;

×

5412

  pInfo->dataVersion = 0;

×

5413

  pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey));

×

5414

  if (!pInfo->historyWins) {

×

5415

    code = terrno;

×

5416

    QUERY_CHECK_CODE(code, lino, _error);

×

5419

  if (pHandle) {

×

5420

    pInfo->isHistoryOp = (pHandle->fillHistory == STREAM_HISTORY_OPERATOR);

×

5423

  code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);

×

5424

  QUERY_CHECK_CODE(code, lino, _error);

×

5426

  pInfo->recvGetAll = false;

×

5427

  pInfo->pPkDeleted = tSimpleHashInit(64, hashFn);

×

5428

  QUERY_CHECK_NULL(pInfo->pPkDeleted, code, lino, _error, terrno);

×

5429

  pInfo->destHasPrimaryKey = pStateNode->window.destHasPrimaryKey;

×

5430

  pInfo->pOperator = pOperator;

×

5432

  setOperatorInfo(pOperator, "StreamStateAggOperator", nodeType(pPhyNode), true, OP_NOT_OPENED,

×

5435

  void*   buff = NULL;

×

5436

  int32_t len = 0;

×

5438

      pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_CHECKPOINT_NAME,

×

5440

  if (res == TSDB_CODE_SUCCESS) {

×

5441

    code = doStreamStateDecodeOpState(buff, len, pOperator, true, NULL);

×

5442

    taosMemoryFree(buff);

×

5443

    QUERY_CHECK_CODE(code, lino, _error);

×

5445

  initNonBlockAggSupptor(&pInfo->nbSup, NULL, NULL);

×

5446

  code = initStreamBasicInfo(&pInfo->basic, pOperator);

×

5447

  QUERY_CHECK_CODE(code, lino, _error);

×

5449

  if (pStateNode->window.triggerType == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {

×

5450

    if (pHandle->fillHistory == STREAM_HISTORY_OPERATOR) {

×

5451

      setFillHistoryOperatorFlag(&pInfo->basic);

×

5452

    } else if (pHandle->fillHistory == STREAM_RECALCUL_OPERATOR) {

×

5453

      setRecalculateOperatorFlag(&pInfo->basic);

×

5457

        createOperatorFpSet(optrDummyOpenFn, doStreamStateNonblockAggNext, NULL, destroyStreamStateOperatorInfo,

×

5459

    setOperatorStreamStateFn(pOperator, streamStateNonblockReleaseState, streamStateNonblockReloadState);

×

5461

    pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAggNext, NULL, destroyStreamStateOperatorInfo,

×

5463

    setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState);

×

5466

  code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,

×

5467

                        &pInfo->twAggSup, &pInfo->basic, 0);

×

5468

  QUERY_CHECK_CODE(code, lino, _error);

×

5470

  code = appendDownstream(pOperator, &downstream, 1);

×

5471

  QUERY_CHECK_CODE(code, lino, _error);

×

5473

  pInfo->trueForLimit = pStateNode->trueForLimit;

×

5475

  *pOptrInfo = pOperator;

×

5476

  return TSDB_CODE_SUCCESS;

×

5478

_error:

×

5479

  if (pInfo != NULL) destroyStreamStateOperatorInfo(pInfo);

×

5480

  destroyOperatorAndDownstreams(pOperator, &downstream, 1);

×

5481

  pTaskInfo->code = code;

×

5482

  qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

5483

  return code;

×

5498

static int32_t doStreamIntervalAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

5499

  int32_t                      code = TSDB_CODE_SUCCESS;

×

5500

  int32_t                      lino = 0;

×

5501

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

5502

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

5503

  SStorageAPI*                 pAPI = &pOperator->pTaskInfo->storageAPI;

×

5504

  SExprSupp*                   pSup = &pOperator->exprSupp;

×

5506

  qDebug("stask:%s  %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);

×

5508

  if (pOperator->status == OP_EXEC_DONE) {

×

5509

    (*ppRes) = NULL;

×

5510

    return code;

×

5513

  if (pOperator->status == OP_RES_TO_RETURN) {

×

5514

    SSDataBlock* resBlock = NULL;

×

5515

    code = buildIntervalResult(pOperator, &resBlock);

×

5516

    QUERY_CHECK_CODE(code, lino, _end);

×

5517

    if (resBlock != NULL) {

×

5518

      (*ppRes) = resBlock;

×

5519

      return code;

×

5522

    if (pInfo->recvGetAll) {

×

5523

      pInfo->recvGetAll = false;

×

5524

      resetUnCloseWinInfo(pInfo->aggSup.pResultRowHashTable);

×

5527

    if (pInfo->reCkBlock) {

×

5528

      pInfo->reCkBlock = false;

×

5529

      printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5530

      (*ppRes) = pInfo->pCheckpointRes;

×

5531

      return code;

×

5534

    setStreamOperatorCompleted(pOperator);

×

5535

    (*ppRes) = NULL;

×

5536

    return code;

×

5539

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

5541

  if (!pInfo->pUpdated) {

×

5542

    pInfo->pUpdated = taosArrayInit(4096, POINTER_BYTES);

×

5543

    QUERY_CHECK_NULL(pInfo->pUpdated, code, lino, _end, terrno);

×

5546

  if (!pInfo->pUpdatedMap) {

×

5547

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

5548

    pInfo->pUpdatedMap = tSimpleHashInit(4096, hashFn);

×

5549

    QUERY_CHECK_NULL(pInfo->pUpdatedMap, code, lino, _end, terrno);

×

5552

  while (1) {

×

5553

    SSDataBlock* pBlock = NULL;

×

5554

    code = downstream->fpSet.getNextFn(downstream, &pBlock);

×

5555

    QUERY_CHECK_CODE(code, lino, _end);

×

5557

    if (pBlock == NULL) {

×

5558

      qDebug("===stream===return data:%s. recv datablock num:%" PRIu64, getStreamOpName(pOperator->operatorType),

×

5560

      pInfo->numOfDatapack = 0;

×

5561

      break;

×

5564

    pInfo->numOfDatapack++;

×

5565

    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));

×

5566

    setStreamOperatorState(&pInfo->basic, pBlock->info.type);

×

5568

    if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||

×

5569

        pBlock->info.type == STREAM_CLEAR) {

×

5570

      code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, pInfo->pDelWins, pInfo->pUpdatedMap, NULL);

×

5571

      QUERY_CHECK_CODE(code, lino, _end);

×

5572

      continue;

×

5573

    } else if (pBlock->info.type == STREAM_GET_ALL) {

×

5574

      pInfo->recvGetAll = true;

×

5575

      code = getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap);

×

5576

      QUERY_CHECK_CODE(code, lino, _end);

×

5577

      continue;

×

5578

    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {

×

5579

      printDataBlock(pBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5580

      (*ppRes) = pBlock;

×

5581

      return code;

×

5582

    } else if (pBlock->info.type == STREAM_DROP_CHILD_TABLE) {

×

5583

      doDeleteWindowByGroupId(pOperator, pBlock);

×

5584

      printDataBlock(pBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5585

      (*ppRes) = pBlock;

×

5586

      return code;

×

5587

    } else if (pBlock->info.type == STREAM_CHECKPOINT) {

×

5588

      pAPI->stateStore.streamStateCommit(pInfo->pState);

×

5589

      doStreamIntervalSaveCheckpoint(pOperator);

×

5590

      pInfo->reCkBlock = true;

×

5591

      code = copyDataBlock(pInfo->pCheckpointRes, pBlock);

×

5592

      QUERY_CHECK_CODE(code, lino, _end);

×

5594

      continue;

×

5596

      if (pBlock->info.type != STREAM_NORMAL && pBlock->info.type != STREAM_INVALID) {

×

5597

        qDebug("===stream===%s ignore recv block. type:%d", GET_TASKID(pTaskInfo), pBlock->info.type);

×

5598

        continue;

×

5602

    if (pBlock->info.type == STREAM_NORMAL && pBlock->info.version != 0) {

×

5604

      pTaskInfo->version = pBlock->info.version;

×

5607

    if (pInfo->scalarSupp.pExprInfo != NULL) {

×

5608

      SExprSupp* pExprSup = &pInfo->scalarSupp;

×

5609

      code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);

×

5610

      QUERY_CHECK_CODE(code, lino, _end);

×

5616

    code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

5617

    QUERY_CHECK_CODE(code, lino, _end);

×

5624

    code = doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pInfo->pUpdatedMap, pInfo->pDeletedMap);

×

5625

    if (code == TSDB_CODE_STREAM_INTERNAL_ERROR) {

×

5626

      pOperator->status = OP_RES_TO_RETURN;

×

5627

      code = TSDB_CODE_SUCCESS;

×

5628

      break;

×

5630

    QUERY_CHECK_CODE(code, lino, _end);

×

5631

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);

×

5632

    pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey);

×

5634

  pOperator->status = OP_RES_TO_RETURN;

×

5635

  if (!pInfo->destHasPrimaryKey) {

×

5636

    removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins);

×

5638

  code = closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL,

×

5640

  QUERY_CHECK_CODE(code, lino, _end);

×

5642

  if (pInfo->destHasPrimaryKey && IS_NORMAL_INTERVAL_OP(pOperator)) {

×

5643

    code = copyIntervalDeleteKey(pInfo->pDeletedMap, pInfo->pDelWins);

×

5644

    QUERY_CHECK_CODE(code, lino, _end);

×

5647

  void*   pIte = NULL;

×

5648

  int32_t iter = 0;

×

5649

  while ((pIte = tSimpleHashIterate(pInfo->pUpdatedMap, pIte, &iter)) != NULL) {

×

5650

    void* tmp = taosArrayPush(pInfo->pUpdated, pIte);

×

5651

    if (!tmp) {

×

5652

      code = terrno;

×

5653

      QUERY_CHECK_CODE(code, lino, _end);

×

5656

  taosArraySort(pInfo->pUpdated, winPosCmprImpl);

×

5658

  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);

×

5659

  pInfo->pUpdated = NULL;

×

5660

  code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);

×

5661

  QUERY_CHECK_CODE(code, lino, _end);

×

5663

  tSimpleHashCleanup(pInfo->pUpdatedMap);

×

5664

  pInfo->pUpdatedMap = NULL;

×

5666

  code = buildIntervalResult(pOperator, ppRes);

×

5667

  QUERY_CHECK_CODE(code, lino, _end);

×

5669

  return code;

×

5671

_end:

×

5672

  if (code != TSDB_CODE_SUCCESS) {

×

5673

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

5674

    pTaskInfo->code = code;

×

5675

    T_LONG_JMP(pTaskInfo->env, code);

×

5677

  setStreamOperatorCompleted(pOperator);

×

5678

  (*ppRes) = NULL;

×

5679

  return code;

×

5682

int32_t createStreamSingleIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,

×

5685

  QRY_PARAM_CHECK(pOptrInfo);

×

5687

  int32_t code = TSDB_CODE_SUCCESS;

×

5688

  int32_t lino = 0;

×

5689

  int32_t numOfCols = 0;

×

5691

  SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo));

×

5692

  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));

×

5693

  if (pInfo == NULL || pOperator == NULL) {

×

5694

    code = terrno;

×

5695

    QUERY_CHECK_CODE(code, lino, _error);

×

5698

  SStreamIntervalPhysiNode* pIntervalPhyNode = (SStreamIntervalPhysiNode*)pPhyNode;

×

5700

  SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);

×

5701

  QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);

×

5702

  initBasicInfo(&pInfo->binfo, pResBlock);

×

5704

  pInfo->interval = (SInterval){

×

5705

      .interval = pIntervalPhyNode->interval,

×

5706

      .sliding = pIntervalPhyNode->sliding,

×

5707

      .intervalUnit = pIntervalPhyNode->intervalUnit,

×

5708

      .slidingUnit = pIntervalPhyNode->slidingUnit,

×

5709

      .offset = pIntervalPhyNode->offset,

×

5710

      .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision,

×

5711

      .timeRange = pIntervalPhyNode->timeRange,

×

5713

  calcIntervalAutoOffset(&pInfo->interval);

×

5715

  pInfo->twAggSup =

×

5716

      (STimeWindowAggSupp){.waterMark = pIntervalPhyNode->window.watermark,

×

5717

                           .calTrigger = pIntervalPhyNode->window.triggerType,

×

5720

                           .deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval)};

×

5722

  pOperator->pTaskInfo = pTaskInfo;

×

5723

  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;

×

5725

  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;

×

5726

  pInfo->ignoreExpiredDataSaved = false;

×

5728

  SExprSupp* pSup = &pOperator->exprSupp;

×

5729

  pSup->hasWindowOrGroup = true;

×

5731

  code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

×

5732

  QUERY_CHECK_CODE(code, lino, _error);

×

5734

  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;

×

5735

  initResultSizeInfo(&pOperator->resultInfo, 4096);

×

5737

  pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState));

×

5738

  QUERY_CHECK_NULL(pInfo->pState, code, lino, _error, terrno);

×

5739

  *(pInfo->pState) = *(pTaskInfo->streamInfo.pState);

×

5740

  pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsIndex);

×

5742

  size_t     keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;

×

5743

  SExprInfo* pExprInfo = NULL;

×

5744

  code = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);

×

5745

  QUERY_CHECK_CODE(code, lino, _error);

×

5746

  code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, pInfo->pState,

×

5748

  QUERY_CHECK_CODE(code, lino, _error);

×

5750

  if (pIntervalPhyNode->window.pExprs != NULL) {

×

5751

    int32_t    numOfScalar = 0;

×

5752

    SExprInfo* pScalarExprInfo = NULL;

×

5754

    code = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalar);

×

5755

    QUERY_CHECK_CODE(code, lino, _error);

×

5757

    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);

×

5758

    QUERY_CHECK_CODE(code, lino, _error);

×

5761

  pInfo->invertible = false;

×

5762

  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));

×

5763

  QUERY_CHECK_NULL(pInfo->pDelWins, code, lino, _error, terrno);

×

5764

  pInfo->delIndex = 0;

×

5766

  code = createSpecialDataBlock(STREAM_DELETE_RESULT, &pInfo->pDelRes);

×

5767

  QUERY_CHECK_CODE(code, lino, _error);

×

5769

  initResultRowInfo(&pInfo->binfo.resultRowInfo);

×

5771

  pInfo->pPhyNode = NULL;  // create new child

×

5772

  pInfo->pPullDataMap = NULL;

×

5773

  pInfo->pFinalPullDataMap = NULL;

×

5774

  pInfo->pPullWins = NULL;  // SPullWindowInfo

×

5775

  pInfo->pullIndex = 0;

×

5776

  pInfo->pPullDataRes = NULL;

×

5777

  pInfo->numOfChild = 0;

×

5778

  pInfo->delKey.ts = INT64_MAX;

×

5779

  pInfo->delKey.groupId = 0;

×

5780

  pInfo->numOfDatapack = 0;

×

5781

  pInfo->pUpdated = NULL;

×

5782

  pInfo->pUpdatedMap = NULL;

×

5783

  int32_t funResSize = getMaxFunResSize(pSup, numOfCols);

×

5785

  pInfo->stateStore = pTaskInfo->storageAPI.stateStore;

×

5786

  pInfo->pState->pFileState = NULL;

×

5789

  pInfo->pState->pResultRowStore.resultRowGet = getResultRowFromBuf;

×

5790

  pInfo->pState->pResultRowStore.resultRowPut = putResultRowToBuf;

×

5791

  pInfo->pState->pExprSupp = &pOperator->exprSupp;

×

5793

  code = pTaskInfo->storageAPI.stateStore.streamFileStateInit(

×

5794

      tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, compareTs, pInfo->pState,

×

5795

      pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo), pHandle->checkpointId, STREAM_STATE_BUFF_HASH,

×

5796

      &pInfo->pState->pFileState);

×

5797

  QUERY_CHECK_CODE(code, lino, _error);

×

5799

  pInfo->pOperator = pOperator;

×

5800

  setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED,

×

5803

      createOperatorFpSet(optrDummyOpenFn, doStreamIntervalAggNext, NULL, destroyStreamFinalIntervalOperatorInfo,

×

5805

  setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState);

×

5807

  code = initStreamBasicInfo(&pInfo->basic, pOperator);

×

5808

  QUERY_CHECK_CODE(code, lino, _error);

×

5810

  pInfo->recvGetAll = false;

×

5812

  code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);

×

5813

  QUERY_CHECK_CODE(code, lino, _error);

×

5815

  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

5816

  pInfo->pDeletedMap = tSimpleHashInit(4096, hashFn);

×

5817

  QUERY_CHECK_NULL(pInfo->pDeletedMap, code, lino, _error, terrno);

×

5818

  pInfo->destHasPrimaryKey = pIntervalPhyNode->window.destHasPrimaryKey;

×

5821

  void*   buff = NULL;

×

5822

  int32_t len = 0;

×

5823

  int32_t res = pAPI->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_CHECKPOINT_NAME,

×

5825

  if (res == TSDB_CODE_SUCCESS) {

×

5826

    doStreamIntervalDecodeOpState(buff, len, pOperator);

×

5827

    taosMemoryFree(buff);

×

5830

  pInfo->basic.primaryPkIndex = -1;

×

5831

  code = initIntervalDownStream(downstream, pPhyNode->type, pInfo, &pInfo->basic);

×

5832

  QUERY_CHECK_CODE(code, lino, _error);

×

5834

  code = appendDownstream(pOperator, &downstream, 1);

×

5835

  QUERY_CHECK_CODE(code, lino, _error);

×

5837

  *pOptrInfo = pOperator;

×

5838

  return TSDB_CODE_SUCCESS;

×

5840

_error:

×

5841

  if (pInfo != NULL) destroyStreamFinalIntervalOperatorInfo(pInfo);

×

5842

  destroyOperatorAndDownstreams(pOperator, &downstream, 1);

×

5843

  pTaskInfo->code = code;

×

5844

  return code;

×

5847

static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pUpdatedMap) {

×

5848

  int32_t                      code = TSDB_CODE_SUCCESS;

×

5849

  int32_t                      lino = 0;

×

5850

  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info;

×

5851

  pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);

×

5853

  SResultRowInfo*  pResultRowInfo = &(pInfo->binfo.resultRowInfo);

×

5854

  SExecTaskInfo*   pTaskInfo = pOperator->pTaskInfo;

×

5855

  SExprSupp*       pSup = &pOperator->exprSupp;

×

5856

  int32_t          numOfOutput = pSup->numOfExprs;

×

5857

  int32_t          step = 1;

×

5858

  SRowBuffPos*     pResPos = NULL;

×

5859

  SResultRow*      pResult = NULL;

×

5860

  int32_t          forwardRows = 1;

×

5861

  uint64_t         groupId = pSDataBlock->info.id.groupId;

×

5862

  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);

×

5863

  TSKEY*           tsCol = (int64_t*)pColDataInfo->pData;

×

5865

  int32_t     startPos = 0;

×

5866

  TSKEY       ts = getStartTsKey(&pSDataBlock->info.window, tsCol);

×

5867

  STimeWindow nextWin = getFinalTimeWindow(ts, &pInfo->interval);

×

5869

  while (1) {

×

5870

    SWinKey key = {

×

5871

        .ts = nextWin.skey,

×

5874

    void*   chIds = taosHashGet(pInfo->pPullDataMap, &key, sizeof(SWinKey));

×

5875

    int32_t index = -1;

×

5876

    SArray* chArray = NULL;

×

5877

    int32_t chId = 0;

×

5878

    if (chIds) {

×

5879

      chArray = *(void**)chIds;

×

5880

      chId = getChildIndex(pSDataBlock);

×

5881

      index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);

×

5883

    if (!(index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA)) {

×

5884

      startPos = getNextQualifiedFinalWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCol, startPos);

×

5885

      if (startPos < 0) {

×

5886

        break;

×

5888

      continue;

×

5891

    if (!inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) {

×

5892

      startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCol, startPos, nextWin.ekey, &nextWin);

×

5893

      if (startPos < 0) {

×

5894

        break;

×

5896

      continue;

×

5899

    int32_t winCode = TSDB_CODE_SUCCESS;

×

5900

    code = setIntervalOutputBuf(pInfo->pState, &nextWin, &pResPos, groupId, pSup->pCtx, numOfOutput,

×

5902

    QUERY_CHECK_CODE(code, lino, _end);

×

5904

    pResult = (SResultRow*)pResPos->pRowBuff;

×

5906

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {

×

5907

      code = saveWinResult(&key, pResPos, pUpdatedMap);

×

5908

      QUERY_CHECK_CODE(code, lino, _end);

×

5911

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

5912

      code = tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pResPos, POINTER_BYTES);

×

5913

      QUERY_CHECK_CODE(code, lino, _end);

×

5916

    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, 1);

×

5917

    code = applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos,

×

5918

                                           forwardRows, pSDataBlock->info.rows, numOfOutput);

×

5919

    QUERY_CHECK_CODE(code, lino, _end);

×

5920

    key.ts = nextWin.skey;

×

5922

    if (pInfo->delKey.ts > key.ts) {

×

5923

      pInfo->delKey = key;

×

5925

    int32_t prevEndPos = (forwardRows - 1) * step + startPos;

×

5926

    if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) {

×

5927

      qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64

×

5930

      code = blockDataUpdateTsWindow(pSDataBlock, 0);

×

5931

      QUERY_CHECK_CODE(code, lino, _end);

×

5934

      if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) {

×

5935

        qError("table uid %" PRIu64 " data block timestamp is out of range! minKey %" PRId64 ",maxKey %" PRId64,

×

5939

    startPos = getNextQualifiedFinalWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCol, prevEndPos);

×

5940

    if (startPos < 0) {

×

5941

      break;

×

5945

_end:

×

5946

  if (code != TSDB_CODE_SUCCESS) {

×

5947

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

5949

×

5951

static int32_t addMidRetriveWindow(SArray* wins, SHashObj* pMidPullMap, int32_t numOfChild) {

×

5952

  int32_t code = TSDB_CODE_SUCCESS;

×

5953

  int32_t lino = 0;

×

5954

  int32_t size = taosArrayGetSize(wins);

×

5955

  for (int32_t i = 0; i < size; i++) {

×

5956

    SWinKey* winKey = taosArrayGet(wins, i);

×

5957

    void*    chIds = taosHashGet(pMidPullMap, winKey, sizeof(SWinKey));

×

5958

    if (!chIds) {

×

5959

      code = addPullWindow(pMidPullMap, winKey, numOfChild);

×

5960

      qDebug("===stream===prepare mid operator retrive for delete %" PRId64 ", size:%d", winKey->ts, numOfChild);

×

5961

      QUERY_CHECK_CODE(code, lino, _end);

×

5964

_end:

×

5965

  if (code != TSDB_CODE_SUCCESS) {

×

5966

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

5968

  return code;

×

5971

static SSDataBlock* buildMidIntervalResult(SOperatorInfo* pOperator) {

×

5972

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

5973

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

5974

  uint16_t                     opType = pOperator->operatorType;

×

5976

  if (pInfo->recvPullover) {

×

5977

    pInfo->recvPullover = false;

×

5978

    printDataBlock(pInfo->pMidPulloverRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5979

    return pInfo->pMidPulloverRes;

×

5982

  qDebug("===stream=== build mid interval result");

×

5983

  doBuildDeleteResult(pTaskInfo, pInfo->pMidPullDatas, &pInfo->midDelIndex, pInfo->pDelRes);

×

5984

  if (pInfo->pDelRes->info.rows != 0) {

×

5986

    printDataBlock(pInfo->pDelRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));

×

5987

    return pInfo->pDelRes;

×

5990

  if (pInfo->recvRetrive) {

×

5991

    pInfo->recvRetrive = false;

×

5992

    printDataBlock(pInfo->pMidRetriveRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5993

    return pInfo->pMidRetriveRes;

×

5996

  return NULL;

×

5999

static int32_t doStreamMidIntervalAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

6000

  int32_t                      code = TSDB_CODE_SUCCESS;

×

6001

  int32_t                      lino = 0;

×

6002

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

6003

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

6004

  SStorageAPI*                 pAPI = &pOperator->pTaskInfo->storageAPI;

×

6005

  SOperatorInfo*               downstream = pOperator->pDownstream[0];

×

6006

  SExprSupp*                   pSup = &pOperator->exprSupp;

×

6008

  qDebug("stask:%s  %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);

×

6010

  if (pOperator->status == OP_EXEC_DONE) {

×

6011

    (*ppRes) = NULL;

×

6012

    return code;

×

6013

  } else if (pOperator->status == OP_RES_TO_RETURN) {

×

6014

    SSDataBlock* resBlock = NULL;

×

6015

    code = buildIntervalResult(pOperator, &resBlock);

×

6016

    QUERY_CHECK_CODE(code, lino, _end);

×

6017

    if (resBlock != NULL) {

×

6018

      (*ppRes) = resBlock;

×

6019

      return code;

×

6022

    setOperatorCompleted(pOperator);

×

6023

    clearFunctionContext(&pOperator->exprSupp);

×

6024

    clearStreamIntervalOperator(pInfo);

×

6025

    qDebug("stask:%s  ===stream===%s clear", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType));

×

6026

    (*ppRes) = NULL;

×

6027

    return code;

×

6029

    SSDataBlock* resBlock = NULL;

×

6030

    code = buildIntervalResult(pOperator, &resBlock);

×

6031

    QUERY_CHECK_CODE(code, lino, _end);

×

6032

    if (resBlock != NULL) {

×

6033

      (*ppRes) = resBlock;

×

6034

      return code;

×

6037

    resBlock = buildMidIntervalResult(pOperator);

×

6038

    if (resBlock != NULL) {

×

6039

      (*ppRes) = resBlock;

×

6040

      return code;

×

6043

    if (pInfo->clearState) {

×

6044

      pInfo->clearState = false;

×

6045

      clearFunctionContext(&pOperator->exprSupp);

×

6046

      clearStreamIntervalOperator(pInfo);

×

6050

  if (!pInfo->pUpdated) {

×

6051

    pInfo->pUpdated = taosArrayInit(4096, POINTER_BYTES);

×

6052

    QUERY_CHECK_NULL(pInfo->pUpdated, code, lino, _end, terrno);

×

6054

  if (!pInfo->pUpdatedMap) {

×

6055

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

6056

    pInfo->pUpdatedMap = tSimpleHashInit(4096, hashFn);

×

6057

    QUERY_CHECK_NULL(pInfo->pUpdatedMap, code, lino, _end, terrno);

×

6060

  while (1) {

×

6061

    if (isTaskKilled(pTaskInfo)) {

×

6062

      qInfo("===stream=== %s task is killed, code %s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code));

×

6063

      (*ppRes) = NULL;

×

6064

      return code;

×

6067

    SSDataBlock* pBlock = NULL;

×

6068

    code = downstream->fpSet.getNextFn(downstream, &pBlock);

×

6069

    QUERY_CHECK_CODE(code, lino, _end);

×

6071

    if (pBlock == NULL) {

×

6072

      pOperator->status = OP_RES_TO_RETURN;

×

6073

      qDebug("===stream===return data:%s. recv datablock num:%" PRIu64, getStreamOpName(pOperator->operatorType),

×

6075

      pInfo->numOfDatapack = 0;

×

6076

      break;

×

6078

    pInfo->numOfDatapack++;

×

6079

    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));

×

6080

    setStreamOperatorState(&pInfo->basic, pBlock->info.type);

×

6082

    if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) {

×

6083

      pInfo->binfo.pRes->info.type = pBlock->info.type;

×

6084

    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||

×

6085

               pBlock->info.type == STREAM_CLEAR) {

×

6086

      SArray* delWins = taosArrayInit(8, sizeof(SWinKey));

×

6087

      if (!delWins) {

×

6088

        code = terrno;

×

6089

        QUERY_CHECK_CODE(code, lino, _end);

×

6092

          doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, pInfo->pFinalPullDataMap);

×

6093

      QUERY_CHECK_CODE(code, lino, _end);

×

6095

      removeResults(delWins, pInfo->pUpdatedMap);

×

6096

      void* tmp = taosArrayAddAll(pInfo->pDelWins, delWins);

×

6097

      if (!tmp && taosArrayGetSize(delWins) > 0) {

×

6098

        code = TSDB_CODE_OUT_OF_MEMORY;

×

6099

        QUERY_CHECK_CODE(code, lino, _end);

×

6101

      taosArrayDestroy(delWins);

×

6103

      doBuildDeleteResult(pTaskInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);

×

6104

      if (pInfo->pDelRes->info.rows != 0) {

×

6106

        printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

6107

        if (pBlock->info.type == STREAM_CLEAR) {

×

6108

          pInfo->pDelRes->info.type = STREAM_CLEAR;

×

6110

          pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;

×

6112

        (*ppRes) = pInfo->pDelRes;

×

6113

        return code;

×

6115

      continue;

×

6116

    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {

×

6117

      (*ppRes) = pBlock;

×

6118

      return code;

×

6119

    } else if (pBlock->info.type == STREAM_PULL_OVER) {

×

6120

      code = processPullOver(pBlock, pInfo->pPullDataMap, pInfo->pFinalPullDataMap, &pInfo->interval, pInfo->pPullWins,

×

6122

      QUERY_CHECK_CODE(code, lino, _end);

×

6124

      if (pInfo->recvPullover) {

×

6125

        code = copyDataBlock(pInfo->pMidPulloverRes, pBlock);

×

6126

        QUERY_CHECK_CODE(code, lino, _end);

×

6128

        pInfo->clearState = true;

×

6129

        break;

×

6131

      continue;

×

6132

    } else if (pBlock->info.type == STREAM_CHECKPOINT) {

×

6133

      pAPI->stateStore.streamStateCommit(pInfo->pState);

×

6134

      doStreamIntervalSaveCheckpoint(pOperator);

×

6135

      code = copyDataBlock(pInfo->pCheckpointRes, pBlock);

×

6136

      QUERY_CHECK_CODE(code, lino, _end);

×

6138

      continue;

×

6139

    } else if (pBlock->info.type == STREAM_MID_RETRIEVE) {

×

6140

      SArray* delWins = taosArrayInit(8, sizeof(SWinKey));

×

6141

      if (!delWins) {

×

6142

        code = terrno;

×

6143

        QUERY_CHECK_CODE(code, lino, _end);

×

6145

      code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, NULL);

×

6146

      QUERY_CHECK_CODE(code, lino, _end);

×

6148

      code = addMidRetriveWindow(delWins, pInfo->pPullDataMap, pInfo->numOfChild);

×

6149

      QUERY_CHECK_CODE(code, lino, _end);

×

6151

      taosArrayDestroy(delWins);

×

6152

      pInfo->recvRetrive = true;

×

6153

      code = copyDataBlock(pInfo->pMidRetriveRes, pBlock);

×

6154

      QUERY_CHECK_CODE(code, lino, _end);

×

6156

      pInfo->pMidRetriveRes->info.type = STREAM_MID_RETRIEVE;

×

6157

      pInfo->clearState = true;

×

6158

      break;

×

6160

      if (pBlock->info.type != STREAM_INVALID) {

×

6161

        code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;

×

6162

        QUERY_CHECK_CODE(code, lino, _end);

×

6166

    if (pInfo->scalarSupp.pExprInfo != NULL) {

×

6167

      SExprSupp* pExprSup = &pInfo->scalarSupp;

×

6168

      code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);

×

6169

      QUERY_CHECK_CODE(code, lino, _end);

×

6171

    code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

6172

    QUERY_CHECK_CODE(code, lino, _end);

×

6173

    doStreamMidIntervalAggImpl(pOperator, pBlock, pInfo->pUpdatedMap);

×

6174

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);

×

6175

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.watermark);

×

6176

    pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey);

×

6179

  removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins);

×

6180

  pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs;

×

6182

  void*   pIte = NULL;

×

6183

  int32_t iter = 0;

×

6184

  while ((pIte = tSimpleHashIterate(pInfo->pUpdatedMap, pIte, &iter)) != NULL) {

×

6185

    void* tmp = taosArrayPush(pInfo->pUpdated, pIte);

×

6186

    if (!tmp) {

×

6187

      code = terrno;

×

6188

      QUERY_CHECK_CODE(code, lino, _end);

×

6192

  tSimpleHashCleanup(pInfo->pUpdatedMap);

×

6193

  pInfo->pUpdatedMap = NULL;

×

6194

  taosArraySort(pInfo->pUpdated, winPosCmprImpl);

×

6196

  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);

×

6197

  pInfo->pUpdated = NULL;

×

6198

  code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);

×

6199

  QUERY_CHECK_CODE(code, lino, _end);

×

6201

  SSDataBlock* resBlock = NULL;

×

6202

  code = buildIntervalResult(pOperator, &resBlock);

×

6203

  QUERY_CHECK_CODE(code, lino, _end);

×

6204

  if (resBlock != NULL) {

×

6205

    (*ppRes) = resBlock;

×

6206

    return code;

×

6209

  resBlock = buildMidIntervalResult(pOperator);

×

6210

  if (resBlock != NULL) {

×

6211

    (*ppRes) = resBlock;

×

6212

    return code;

×

6215

  if (pInfo->clearState) {

×

6216

    pInfo->clearState = false;

×

6217

    clearFunctionContext(&pOperator->exprSupp);

×

6218

    clearStreamIntervalOperator(pInfo);

×

6221

_end:

×

6222

  if (code != TSDB_CODE_SUCCESS) {

×

6223

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

6224

    pTaskInfo->code = code;

×

6225

    T_LONG_JMP(pTaskInfo->env, code);

×

6227

  (*ppRes) = NULL;

×

6228

  return code;

×

6231

static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) {

×

6232

  SSDataBlock* pRes = NULL;

×

6233

  int32_t      code = doStreamMidIntervalAggNext(pOperator, &pRes);

×

6234

  return pRes;

×

6237

void setStreamOperatorCompleted(SOperatorInfo* pOperator) {

×

6238

  qDebug("stask:%s  %s status: %d. set completed", GET_TASKID(pOperator->pTaskInfo),

×

6240

  setOperatorCompleted(pOperator);

×

6241

×

taosdata / TDengine / #3780

Source File
Press 'n' to go to next uncovered line, 'b' for previous

taosdata / TDengine / #3780

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous