taosdata/TDengine | Build 4143 | source/libs/executor/src/streamtimewindowoperator.c | Coveralls

50

int32_t binarySearchCom(void* keyList, int num, void* pKey, int order, __compare_fn_t comparefn) {

×

51

  int firstPos = 0, lastPos = num - 1, midPos = -1;

×

52

  int numOfRows = 0;

×

54

  if (num <= 0) return -1;

×

55

  if (order == TSDB_ORDER_DESC) {

×

58

      if (comparefn(pKey, keyList, lastPos) >= 0) return lastPos;

×

59

      if (comparefn(pKey, keyList, firstPos) == 0) return firstPos;

×

60

      if (comparefn(pKey, keyList, firstPos) < 0) return firstPos - 1;

×

62

      numOfRows = lastPos - firstPos + 1;

×

63

      midPos = (numOfRows >> 1) + firstPos;

×

65

      if (comparefn(pKey, keyList, midPos) < 0) {

×

66

        lastPos = midPos - 1;

×

67

      } else if (comparefn(pKey, keyList, midPos) > 0) {

×

68

        firstPos = midPos + 1;

×

70

        break;

×

77

      if (comparefn(pKey, keyList, firstPos) <= 0) return firstPos;

×

78

      if (comparefn(pKey, keyList, lastPos) == 0) return lastPos;

×

80

      if (comparefn(pKey, keyList, lastPos) > 0) {

×

81

        lastPos = lastPos + 1;

×

82

        if (lastPos >= num)

×

83

          return -1;

×

85

          return lastPos;

×

88

      numOfRows = lastPos - firstPos + 1;

×

89

      midPos = (numOfRows >> 1) + firstPos;

×

91

      if (comparefn(pKey, keyList, midPos) < 0) {

×

92

        lastPos = midPos - 1;

×

93

      } else if (comparefn(pKey, keyList, midPos) > 0) {

×

94

        firstPos = midPos + 1;

×

96

        break;

×

101

  return midPos;

×

104

static int32_t comparePullWinKey(void* pKey, void* data, int32_t index) {

×

105

  SArray*          res = (SArray*)data;

×

106

  SPullWindowInfo* pos = taosArrayGet(res, index);

×

107

  SPullWindowInfo* pData = (SPullWindowInfo*)pKey;

×

108

  if (pData->groupId > pos->groupId) {

×

109

    return 1;

×

110

  } else if (pData->groupId < pos->groupId) {

×

111

    return -1;

×

114

  if (pData->window.skey > pos->window.ekey) {

×

115

    return 1;

×

116

  } else if (pData->window.ekey < pos->window.skey) {

×

117

    return -1;

×

119

  return 0;

×

122

static int32_t savePullWindow(SPullWindowInfo* pPullInfo, SArray* pPullWins) {

×

123

  int32_t size = taosArrayGetSize(pPullWins);

×

124

  int32_t index = binarySearchCom(pPullWins, size, pPullInfo, TSDB_ORDER_DESC, comparePullWinKey);

×

125

  if (index == -1) {

×

126

    index = 0;

×

128

    int32_t code = comparePullWinKey(pPullInfo, pPullWins, index);

×

129

    if (code == 0) {

×

130

      SPullWindowInfo* pos = taosArrayGet(pPullWins, index);

×

131

      pos->window.skey = TMIN(pos->window.skey, pPullInfo->window.skey);

×

132

      pos->window.ekey = TMAX(pos->window.ekey, pPullInfo->window.ekey);

×

133

      pos->calWin.skey = TMIN(pos->calWin.skey, pPullInfo->calWin.skey);

×

134

      pos->calWin.ekey = TMAX(pos->calWin.ekey, pPullInfo->calWin.ekey);

×

135

      return TSDB_CODE_SUCCESS;

×

136

    } else if (code > 0) {

×

137

      index++;

×

140

  if (taosArrayInsert(pPullWins, index, pPullInfo) == NULL) {

×

141

    return terrno;

×

143

  return TSDB_CODE_SUCCESS;

×

146

int32_t saveResult(SResultWindowInfo winInfo, SSHashObj* pStUpdated) {

×

147

  if (tSimpleHashGetSize(pStUpdated) > MAX_STREAM_HISTORY_RESULT) {

×

148

    qError("%s failed at line %d since too many history result. ", __func__, __LINE__);

×

149

    return TSDB_CODE_STREAM_INTERNAL_ERROR;

×

151

  winInfo.sessionWin.win.ekey = winInfo.sessionWin.win.skey;

×

152

  return tSimpleHashPut(pStUpdated, &winInfo.sessionWin, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo));

×

155

int32_t saveWinResult(SWinKey* pKey, SRowBuffPos* pPos, SSHashObj* pUpdatedMap) {

×

156

  if (tSimpleHashGetSize(pUpdatedMap) > MAX_STREAM_HISTORY_RESULT) {

×

157

    qError("%s failed at line %d since too many history result. ", __func__, __LINE__);

×

158

    return TSDB_CODE_STREAM_INTERNAL_ERROR;

×

160

  return tSimpleHashPut(pUpdatedMap, pKey, sizeof(SWinKey), &pPos, POINTER_BYTES);

×

163

static int32_t saveWinResultInfo(TSKEY ts, uint64_t groupId, SRowBuffPos* pPos, SSHashObj* pUpdatedMap) {

×

164

  SWinKey key = {.ts = ts, .groupId = groupId};

×

165

  return saveWinResult(&key, pPos, pUpdatedMap);

×

168

static void removeResults(SArray* pWins, SSHashObj* pUpdatedMap) {

×

169

  int32_t size = taosArrayGetSize(pWins);

×

170

  for (int32_t i = 0; i < size; i++) {

×

171

    SWinKey* pW = taosArrayGet(pWins, i);

×

172

    void*    tmp = tSimpleHashGet(pUpdatedMap, pW, sizeof(SWinKey));

×

173

    if (tmp) {

×

174

      void* value = *(void**)tmp;

×

175

      taosMemoryFree(value);

×

176

      int32_t tmpRes = tSimpleHashRemove(pUpdatedMap, pW, sizeof(SWinKey));

×

177

      qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

180

×

182

int32_t compareWinKey(void* pKey, void* data, int32_t index) {

×

183

  void* pDataPos = taosArrayGet((SArray*)data, index);

×

184

  return winKeyCmprImpl(pKey, pDataPos);

×

187

void removeDeleteResults(SSHashObj* pUpdatedMap, SArray* pDelWins) {

×

188

  taosArraySort(pDelWins, winKeyCmprImpl);

×

189

  taosArrayRemoveDuplicate(pDelWins, winKeyCmprImpl, NULL);

×

190

  int32_t delSize = taosArrayGetSize(pDelWins);

×

191

  if (tSimpleHashGetSize(pUpdatedMap) == 0 || delSize == 0) {

×

192

    return;

×

194

  void*   pIte = NULL;

×

195

  int32_t iter = 0;

×

196

  while ((pIte = tSimpleHashIterate(pUpdatedMap, pIte, &iter)) != NULL) {

×

197

    SWinKey* pResKey = tSimpleHashGetKey(pIte, NULL);

×

198

    int32_t  index = binarySearchCom(pDelWins, delSize, pResKey, TSDB_ORDER_DESC, compareWinKey);

×

199

    if (index >= 0 && 0 == compareWinKey(pResKey, pDelWins, index)) {

×

200

      taosArrayRemove(pDelWins, index);

×

201

      delSize = taosArrayGetSize(pDelWins);

×

206

bool isOverdue(TSKEY ekey, STimeWindowAggSupp* pTwSup) {

×

207

  return pTwSup->maxTs != INT64_MIN && ekey < pTwSup->maxTs - pTwSup->waterMark;

×

210

bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pTwSup) { return isOverdue(pWin->ekey, pTwSup); }

×

212

static void doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId) {

×

213

  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;

×

215

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

216

  SWinKey                      key = {.ts = ts, .groupId = groupId};

×

217

  int32_t                      tmpRes = tSimpleHashRemove(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey));

×

218

  qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

219

  pAPI->stateStore.streamStateDel(pInfo->pState, &key);

×

220

×

222

int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; }

×

224

static void doDeleteWindowByGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock) {

×

225

  SStorageAPI*                 pAPI = &pOperator->pTaskInfo->storageAPI;

×

226

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

228

  SColumnInfoData* pGpIdCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);

×

229

  uint64_t* pGroupIdData = (uint64_t*)pGpIdCol->pData;

×

230

  for (int32_t i = 0; i < pBlock->info.rows; i++) {

×

231

    uint64_t groupId = pGroupIdData[i];

×

232

    void*   pIte = NULL;

×

233

    int32_t iter = 0;

×

234

    while ((pIte = tSimpleHashIterate(pInfo->aggSup.pResultRowHashTable, pIte, &iter)) != NULL) {

×

235

      size_t keyLen = 0;

×

236

      SWinKey* pKey = tSimpleHashGetKey(pIte, &keyLen);

×

237

      if (pKey->groupId == groupId) {

×

238

        int32_t tmpRes = tSimpleHashIterateRemove(pInfo->aggSup.pResultRowHashTable, pKey, keyLen, &pIte, &iter);

×

239

        qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

243

    pAPI->stateStore.streamStateDelByGroupId(pInfo->pState, groupId);

×

245

×

247

static int32_t doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDataBlock* pBlock, SArray* pUpWins,

×

249

  int32_t                      code = TSDB_CODE_SUCCESS;

×

250

  int32_t                      lino = 0;

×

251

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

252

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

253

  SColumnInfoData*             pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);

×

254

  TSKEY*                       startTsCols = (TSKEY*)pStartTsCol->pData;

×

255

  SColumnInfoData*             pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);

×

256

  TSKEY*                       endTsCols = (TSKEY*)pEndTsCol->pData;

×

257

  SColumnInfoData*             pCalStTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);

×

258

  TSKEY*                       calStTsCols = (TSKEY*)pCalStTsCol->pData;

×

259

  SColumnInfoData*             pCalEnTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);

×

260

  TSKEY*                       calEnTsCols = (TSKEY*)pCalEnTsCol->pData;

×

261

  SColumnInfoData*             pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);

×

262

  uint64_t*                    pGpDatas = (uint64_t*)pGpCol->pData;

×

263

  for (int32_t i = 0; i < pBlock->info.rows; i++) {

×

264

    SResultRowInfo dumyInfo = {0};

×

265

    dumyInfo.cur.pageId = -1;

×

267

    STimeWindow win = {0};

×

268

    if (IS_FINAL_INTERVAL_OP(pOperator) || IS_MID_INTERVAL_OP(pOperator)) {

×

269

      win.skey = startTsCols[i];

×

270

      win.ekey = endTsCols[i];

×

272

      win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC);

×

276

      if (!inCalSlidingWindow(pInterval, &win, calStTsCols[i], calEnTsCols[i], pBlock->info.type)) {

×

277

        getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC);

×

278

        continue;

×

280

      uint64_t winGpId = pGpDatas[i];

×

281

      SWinKey  winRes = {.ts = win.skey, .groupId = winGpId};

×

282

      void*    chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey));

×

283

      if (chIds) {

×

284

        int32_t childId = getChildIndex(pBlock);

×

285

        if (pInvalidWins) {

×

286

          qDebug("===stream===save invalid delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d", winRes.ts,

×

288

          code = taosHashPut(pInvalidWins, &winRes, sizeof(SWinKey), NULL, 0);

×

289

          QUERY_CHECK_CODE(code, lino, _end);

×

292

        SArray* chArray = *(void**)chIds;

×

293

        int32_t index = taosArraySearchIdx(chArray, &childId, compareInt32Val, TD_EQ);

×

294

        if (index != -1) {

×

295

          qDebug("===stream===try push delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d ,continue", win.skey,

×

297

          getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC);

×

298

          continue;

×

301

      doDeleteWindow(pOperator, win.skey, winGpId);

×

302

      if (pUpWins) {

×

303

        void* tmp = taosArrayPush(pUpWins, &winRes);

×

304

        if (!tmp) {

×

305

          code = terrno;

×

306

          QUERY_CHECK_CODE(code, lino, _end);

×

309

      if (pUpdatedMap) {

×

310

        int32_t tmpRes = tSimpleHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey));

×

311

        qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

313

      getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC);

×

314

    } while (win.ekey <= endTsCols[i]);

×

316

_end:

×

317

  if (code != TSDB_CODE_SUCCESS) {

×

318

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

320

  return code;

×

323

static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SSHashObj* resWins) {

×

324

  void*   pIte = NULL;

×

325

  int32_t iter = 0;

×

326

  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {

×

327

    SWinKey*     pKey = tSimpleHashGetKey(pIte, NULL);

×

328

    uint64_t     groupId = pKey->groupId;

×

329

    TSKEY        ts = pKey->ts;

×

330

    SRowBuffPos* pPos = *(SRowBuffPos**)pIte;

×

331

    if (!pPos->beUpdated) {

×

332

      continue;

×

334

    pPos->beUpdated = false;

×

335

    int32_t code = saveWinResultInfo(ts, groupId, pPos, resWins);

×

336

    if (code != TSDB_CODE_SUCCESS) {

×

337

      return code;

×

340

  return TSDB_CODE_SUCCESS;

×

343

static int32_t closeStreamIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SInterval* pInterval,

×

346

  qDebug("===stream===close interval window");

×

347

  int32_t                      code = TSDB_CODE_SUCCESS;

×

348

  int32_t                      lino = 0;

×

349

  void*                        pIte = NULL;

×

350

  int32_t                      iter = 0;

×

351

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

352

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

353

  int32_t                      delSize = taosArrayGetSize(pDelWins);

×

354

  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {

×

355

    void*    key = tSimpleHashGetKey(pIte, NULL);

×

356

    SWinKey* pWinKey = (SWinKey*)key;

×

357

    if (delSize > 0) {

×

358

      int32_t index = binarySearchCom(pDelWins, delSize, pWinKey, TSDB_ORDER_DESC, compareWinKey);

×

359

      if (index >= 0 && 0 == compareWinKey(pWinKey, pDelWins, index)) {

×

360

        taosArrayRemove(pDelWins, index);

×

361

        delSize = taosArrayGetSize(pDelWins);

×

365

    void*       chIds = taosHashGet(pPullDataMap, pWinKey, sizeof(SWinKey));

×

366

    STimeWindow win = {

×

367

        .skey = pWinKey->ts,

×

368

        .ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision, NULL) - 1,

×

370

    if (isCloseWindow(&win, pTwSup)) {

×

371

      if (chIds && pPullDataMap) {

×

372

        SArray* chAy = *(SArray**)chIds;

×

373

        int32_t size = taosArrayGetSize(chAy);

×

374

        qDebug("===stream===window %" PRId64 " wait child size:%d", pWinKey->ts, size);

×

375

        for (int32_t i = 0; i < size; i++) {

×

376

          qDebug("===stream===window %" PRId64 " wait child id:%d", pWinKey->ts, *(int32_t*)taosArrayGet(chAy, i));

×

378

        continue;

×

379

      } else if (pPullDataMap) {

×

380

        qDebug("===stream===close window %" PRId64, pWinKey->ts);

×

383

      if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

384

        code = saveWinResult(pWinKey, *(SRowBuffPos**)pIte, closeWins);

×

385

        QUERY_CHECK_CODE(code, lino, _end);

×

387

      int32_t tmpRes = tSimpleHashIterateRemove(pHashMap, pWinKey, sizeof(SWinKey), &pIte, &iter);

×

388

      qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

392

_end:

×

393

  if (code != TSDB_CODE_SUCCESS) {

×

394

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

396

  return code;

×

399

STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) {

×

400

  STimeWindow w = {.skey = ts, .ekey = INT64_MAX};

×

401

  w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision, NULL) - 1;

×

402

  return w;

×

405

static void doBuildDeleteResult(SExecTaskInfo* pTaskInfo, SArray* pWins, int32_t* index,

×

407

  doBuildDeleteResultImpl(&pTaskInfo->storageAPI.stateStore, pTaskInfo->streamInfo.pState, pWins, index, pBlock);

×

408

×

410

void doBuildDeleteResultImpl(SStateStore* pAPI, SStreamState* pState, SArray* pWins, int32_t* index,

×

412

  int32_t code = TSDB_CODE_SUCCESS;

×

413

  int32_t lino = 0;

×

414

  blockDataCleanup(pBlock);

×

415

  int32_t size = taosArrayGetSize(pWins);

×

416

  if (*index == size) {

×

417

    *index = 0;

×

418

    taosArrayClear(pWins);

×

419

    goto _end;

×

421

  code = blockDataEnsureCapacity(pBlock, size - *index);

×

422

  QUERY_CHECK_CODE(code, lino, _end);

×

424

  uint64_t uid = 0;

×

425

  for (int32_t i = *index; i < size; i++) {

×

426

    SWinKey* pWin = taosArrayGet(pWins, i);

×

427

    void*    tbname = NULL;

×

428

    int32_t  winCode = TSDB_CODE_SUCCESS;

×

429

    code = pAPI->streamStateGetParName(pState, pWin->groupId, &tbname, false, &winCode);

×

430

    QUERY_CHECK_CODE(code, lino, _end);

×

432

    if (winCode != TSDB_CODE_SUCCESS) {

×

433

      code = appendDataToSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, NULL);

×

434

      QUERY_CHECK_CODE(code, lino, _end);

×

436

      QUERY_CHECK_CONDITION((tbname), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);

×

438

      STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));

×

439

      code = appendDataToSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, parTbName);

×

440

      QUERY_CHECK_CODE(code, lino, _end);

×

442

    pAPI->streamStateFreeVal(tbname);

×

443

    (*index)++;

×

446

_end:

×

447

  if (code != TSDB_CODE_SUCCESS) {

×

448

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

450

×

452

void destroyFlusedPos(void* pRes) {

×

453

  SRowBuffPos* pPos = (SRowBuffPos*)pRes;

×

454

  if (pPos->needFree && !pPos->pRowBuff) {

×

455

    taosMemoryFreeClear(pPos->pKey);

×

456

    taosMemoryFree(pPos);

×

458

×

460

void destroyFlusedppPos(void* ppRes) {

×

461

  void* pRes = *(void**)ppRes;

×

462

  destroyFlusedPos(pRes);

×

463

×

465

void clearGroupResInfo(SGroupResInfo* pGroupResInfo) {

×

466

  int32_t size = taosArrayGetSize(pGroupResInfo->pRows);

×

467

  if (pGroupResInfo->index >= 0 && pGroupResInfo->index < size) {

×

468

    for (int32_t i = pGroupResInfo->index; i < size; i++) {

×

469

      void* pPos = taosArrayGetP(pGroupResInfo->pRows, i);

×

470

      destroyFlusedPos(pPos);

×

474

  pGroupResInfo->freeItem = false;

×

475

  taosArrayDestroy(pGroupResInfo->pRows);

×

476

  pGroupResInfo->pRows = NULL;

×

477

  pGroupResInfo->index = 0;

×

478

  pGroupResInfo->delIndex = 0;

×

479

×

481

void destroyStreamFinalIntervalOperatorInfo(void* param) {

×

482

  if (param == NULL) {

×

483

    return;

×

485

  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)param;

×

486

  cleanupBasicInfo(&pInfo->binfo);

×

487

  if (pInfo->pOperator) {

×

488

    cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, &pInfo->groupResInfo, &pInfo->aggSup,

×

490

    pInfo->pOperator = NULL;

×

493

  destroyStreamBasicInfo(&pInfo->basic);

×

494

  cleanupAggSup(&pInfo->aggSup);

×

495

  clearGroupResInfo(&pInfo->groupResInfo);

×

496

  taosArrayDestroyP(pInfo->pUpdated, destroyFlusedPos);

×

497

  pInfo->pUpdated = NULL;

×

500

  void* pIte = NULL;

×

501

  while ((pIte = taosHashIterate(pInfo->pPullDataMap, pIte)) != NULL) {

×

502

    taosArrayDestroy(*(void**)pIte);

×

504

  taosHashCleanup(pInfo->pPullDataMap);

×

505

  taosHashCleanup(pInfo->pFinalPullDataMap);

×

506

  taosArrayDestroy(pInfo->pPullWins);

×

507

  blockDataDestroy(pInfo->pPullDataRes);

×

508

  taosArrayDestroy(pInfo->pDelWins);

×

509

  blockDataDestroy(pInfo->pDelRes);

×

510

  blockDataDestroy(pInfo->pMidRetriveRes);

×

511

  blockDataDestroy(pInfo->pMidPulloverRes);

×

512

  if (pInfo->pUpdatedMap != NULL) {

×

514

    tSimpleHashSetFreeFp(pInfo->pUpdatedMap, destroyFlusedppPos);

×

515

    tSimpleHashCleanup(pInfo->pUpdatedMap);

×

516

    pInfo->pUpdatedMap = NULL;

×

519

  if (pInfo->stateStore.streamFileStateDestroy != NULL) {

×

520

    pInfo->stateStore.streamFileStateDestroy(pInfo->pState->pFileState);

×

522

  taosArrayDestroy(pInfo->pMidPullDatas);

×

524

  if (pInfo->pState != NULL && pInfo->pState->dump == 1) {

×

525

    taosMemoryFreeClear(pInfo->pState->pTdbState->pOwner);

×

526

    taosMemoryFreeClear(pInfo->pState->pTdbState);

×

528

  taosMemoryFreeClear(pInfo->pState);

×

530

  nodesDestroyNode((SNode*)pInfo->pPhyNode);

×

531

  colDataDestroy(&pInfo->twAggSup.timeWindowData);

×

532

  cleanupExprSupp(&pInfo->scalarSupp);

×

533

  tSimpleHashCleanup(pInfo->pDeletedMap);

×

535

  blockDataDestroy(pInfo->pCheckpointRes);

×

537

  taosMemoryFreeClear(param);

×

551

void reloadFromDownStream(SOperatorInfo* downstream, SStreamIntervalOperatorInfo* pInfo) {

×

552

  SStateStore* pAPI = &downstream->pTaskInfo->storageAPI.stateStore;

×

554

  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {

×

555

    reloadFromDownStream(downstream->pDownstream[0], pInfo);

×

556

    return;

×

559

  SStreamScanInfo* pScanInfo = downstream->info;

×

560

  pInfo->pUpdateInfo = pScanInfo->pUpdateInfo;

×

563

bool hasSrcPrimaryKeyCol(SSteamOpBasicInfo* pInfo) { return pInfo->primaryPkIndex != -1; }

×

565

int32_t initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamIntervalOperatorInfo* pInfo,

×

567

  int32_t        code = TSDB_CODE_SUCCESS;

×

568

  int32_t        lino = 0;

×

569

  SStateStore*   pAPI = &downstream->pTaskInfo->storageAPI.stateStore;

×

570

  SExecTaskInfo* pTaskInfo = downstream->pTaskInfo;

×

572

  if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) {

×

573

    SStreamPartitionOperatorInfo* pScanInfo = downstream->info;

×

574

    pBasic->primaryPkIndex = pScanInfo->basic.primaryPkIndex;

×

577

  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {

×

578

    return initIntervalDownStream(downstream->pDownstream[0], type, pInfo, pBasic);

×

581

  SStreamScanInfo* pScanInfo = downstream->info;

×

582

  pScanInfo->windowSup.parentType = type;

×

583

  pScanInfo->windowSup.pIntervalAggSup = &pInfo->aggSup;

×

584

  if (!pScanInfo->pUpdateInfo) {

×

585

    code = pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate,

×

586

                                 pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo);

×

587

    QUERY_CHECK_CODE(code, lino, _end);

×

590

  pScanInfo->interval = pInfo->interval;

×

591

  pScanInfo->twAggSup = pInfo->twAggSup;

×

592

  pScanInfo->pState = pInfo->pState;

×

593

  pInfo->pUpdateInfo = pScanInfo->pUpdateInfo;

×

594

  if (!hasSrcPrimaryKeyCol(pBasic)) {

×

595

    pBasic->primaryPkIndex = pScanInfo->basic.primaryPkIndex;

×

598

_end:

×

599

  if (code != TSDB_CODE_SUCCESS) {

×

600

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

602

  return code;

×

605

int32_t compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput,

×

607

  int32_t code = TSDB_CODE_SUCCESS;

×

608

  int32_t lino = 0;

×

609

  for (int32_t k = 0; k < numOfOutput; ++k) {

×

610

    if (fmIsWindowPseudoColumnFunc(pDestCtx[k].functionId)) {

×

611

      if (!pTimeWindowData) {

×

612

        continue;

×

615

      SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pDestCtx[k]);

×

616

      char*                p = GET_ROWCELL_INTERBUF(pEntryInfo);

×

617

      SColumnInfoData      idata = {0};

×

618

      idata.info.type = TSDB_DATA_TYPE_BIGINT;

×

619

      idata.info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes;

×

620

      idata.pData = p;

×

622

      SScalarParam out = {.columnData = &idata};

×

623

      SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData};

×

624

      code = pDestCtx[k].sfp.process(&tw, 1, &out);

×

625

      QUERY_CHECK_CODE(code, lino, _end);

×

627

      pEntryInfo->numOfRes = 1;

×

628

    } else if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) {

×

629

      code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]);

×

630

      QUERY_CHECK_CODE(code, lino, _end);

×

631

    } else if (pDestCtx[k].fpSet.combine == NULL) {

×

632

      char* funName = fmGetFuncName(pDestCtx[k].functionId);

×

633

      qError("%s error, combine funcion for %s is not implemented", GET_TASKID(pTaskInfo), funName);

×

634

      taosMemoryFreeClear(funName);

×

635

      code = TSDB_CODE_FAILED;

×

636

      QUERY_CHECK_CODE(code, lino, _end);

×

640

_end:

×

641

  if (code != TSDB_CODE_SUCCESS) {

×

642

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

644

  return code;

×

647

bool hasIntervalWindow(void* pState, SWinKey* pKey, SStateStore* pStore) {

×

648

  return pStore->streamStateCheck(pState, pKey, false, NULL);

×

651

int32_t setIntervalOutputBuf(void* pState, STimeWindow* win, SRowBuffPos** pResult, int64_t groupId,

×

654

  int32_t code = TSDB_CODE_SUCCESS;

×

655

  int32_t lino = 0;

×

656

  SWinKey key = {.ts = win->skey, .groupId = groupId};

×

657

  char*   value = NULL;

×

658

  int32_t size = pAggSup->resultRowSize;

×

660

  code = pStore->streamStateAddIfNotExist(pState, &key, (void**)&value, &size, pWinCode);

×

661

  QUERY_CHECK_CODE(code, lino, _end);

×

663

  *pResult = (SRowBuffPos*)value;

×

664

  SResultRow* res = (SResultRow*)((*pResult)->pRowBuff);

×

667

  res->win = (*win);

×

668

  code = setResultRowInitCtx(res, pCtx, numOfOutput, rowEntryInfoOffset);

×

669

  QUERY_CHECK_CODE(code, lino, _end);

×

671

_end:

×

672

  if (code != TSDB_CODE_SUCCESS) {

×

673

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

675

  return code;

×

678

bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, STimeWindowAggSupp* pTwSup,

×

680

  if (pTwSup->maxTs != INT64_MIN && pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark) {

×

681

    SWinKey key = {.ts = pWin->skey, .groupId = groupId};

×

682

    if (!hasIntervalWindow(pState, &key, pStore)) {

×

683

      return true;

×

685

    return false;

×

687

  return false;

×

690

int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols, int32_t startPos, TSKEY eKey,

×

693

      getNumOfRowsInTimeWindow(pBlockInfo, tsCols, startPos, eKey, binarySearchForKey, NULL, TSDB_ORDER_ASC);

×

694

  int32_t prevEndPos = forwardRows - 1 + startPos;

×

695

  return getNextQualifiedWindow(pInterval, pNextWin, pBlockInfo, tsCols, prevEndPos, TSDB_ORDER_ASC);

×

698

int32_t addPullWindow(SHashObj* pMap, SWinKey* pWinRes, int32_t size) {

×

699

  int32_t code = TSDB_CODE_SUCCESS;

×

700

  int32_t lino = 0;

×

701

  SArray* childIds = taosArrayInit(8, sizeof(int32_t));

×

702

  QUERY_CHECK_NULL(childIds, code, lino, _end, terrno);

×

703

  for (int32_t i = 0; i < size; i++) {

×

704

    void* tmp = taosArrayPush(childIds, &i);

×

705

    if (!tmp) {

×

706

      code = terrno;

×

707

      QUERY_CHECK_CODE(code, lino, _end);

×

710

  code = taosHashPut(pMap, pWinRes, sizeof(SWinKey), &childIds, sizeof(void*));

×

711

  QUERY_CHECK_CODE(code, lino, _end);

×

712

_end:

×

713

  if (code != TSDB_CODE_SUCCESS) {

×

714

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

716

  return code;

×

719

static void clearStreamIntervalOperator(SStreamIntervalOperatorInfo* pInfo) {

×

720

  tSimpleHashClear(pInfo->aggSup.pResultRowHashTable);

×

721

  clearDiskbasedBuf(pInfo->aggSup.pResultBuf);

×

722

  initResultRowInfo(&pInfo->binfo.resultRowInfo);

×

723

  pInfo->aggSup.currentPageId = -1;

×

724

  pInfo->stateStore.streamStateClear(pInfo->pState);

×

725

×

727

static void clearSpecialDataBlock(SSDataBlock* pBlock) {

×

728

  if (pBlock->info.rows <= 0) {

×

729

    return;

×

731

  blockDataCleanup(pBlock);

×

734

void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pBlock) {

×

735

  int32_t code = TSDB_CODE_SUCCESS;

×

736

  int32_t lino = 0;

×

737

  clearSpecialDataBlock(pBlock);

×

738

  int32_t size = taosArrayGetSize(array);

×

739

  if (size - (*pIndex) == 0) {

×

740

    goto _end;

×

742

  code = blockDataEnsureCapacity(pBlock, size - (*pIndex));

×

743

  QUERY_CHECK_CODE(code, lino, _end);

×

745

  SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);

×

746

  SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);

×

747

  SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);

×

748

  SColumnInfoData* pCalStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);

×

749

  SColumnInfoData* pCalEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);

×

750

  SColumnInfoData* pTbName = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);

×

751

  SColumnInfoData* pPrimaryKey = NULL;

×

752

  if (taosArrayGetSize(pBlock->pDataBlock) > PRIMARY_KEY_COLUMN_INDEX) {

×

753

    pPrimaryKey = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, PRIMARY_KEY_COLUMN_INDEX);

×

755

  for (; (*pIndex) < size; (*pIndex)++) {

×

756

    SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex));

×

757

    code = colDataSetVal(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false);

×

758

    QUERY_CHECK_CODE(code, lino, _end);

×

760

    code = colDataSetVal(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false);

×

761

    QUERY_CHECK_CODE(code, lino, _end);

×

763

    code = colDataSetVal(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false);

×

764

    QUERY_CHECK_CODE(code, lino, _end);

×

766

    code = colDataSetVal(pCalStartTs, pBlock->info.rows, (const char*)&pWin->calWin.skey, false);

×

767

    QUERY_CHECK_CODE(code, lino, _end);

×

769

    code = colDataSetVal(pCalEndTs, pBlock->info.rows, (const char*)&pWin->calWin.ekey, false);

×

770

    QUERY_CHECK_CODE(code, lino, _end);

×

772

    colDataSetNULL(pTbName, pBlock->info.rows);

×

773

    if (pPrimaryKey != NULL) {

×

774

      colDataSetNULL(pPrimaryKey, pBlock->info.rows);

×

777

    pBlock->info.rows++;

×

779

  if ((*pIndex) == size) {

×

780

    *pIndex = 0;

×

781

    taosArrayClear(array);

×

783

  code = blockDataUpdateTsWindow(pBlock, 0);

×

784

  QUERY_CHECK_CODE(code, lino, _end);

×

786

_end:

×

787

  if (code != TSDB_CODE_SUCCESS) {

×

788

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

790

×

792

static int32_t processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval,

×

794

  int32_t                      code = TSDB_CODE_SUCCESS;

×

795

  int32_t                      lino = 0;

×

796

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

797

  SColumnInfoData*             pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);

×

798

  TSKEY*                       tsData = (TSKEY*)pStartCol->pData;

×

799

  SColumnInfoData*             pEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);

×

800

  TSKEY*                       tsEndData = (TSKEY*)pEndCol->pData;

×

801

  SColumnInfoData*             pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);

×

802

  uint64_t*                    groupIdData = (uint64_t*)pGroupCol->pData;

×

803

  int32_t                      chId = getChildIndex(pBlock);

×

804

  bool                         res = false;

×

805

  for (int32_t i = 0; i < pBlock->info.rows; i++) {

×

806

    TSKEY winTs = tsData[i];

×

807

    while (winTs <= tsEndData[i]) {

×

808

      SWinKey winRes = {.ts = winTs, .groupId = groupIdData[i]};

×

809

      void*   chIds = taosHashGet(pMap, &winRes, sizeof(SWinKey));

×

810

      if (chIds) {

×

811

        SArray* chArray = *(SArray**)chIds;

×

812

        int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);

×

813

        if (index != -1) {

×

814

          qDebug("===stream===retrive window %" PRId64 " delete child id %d", winRes.ts, chId);

×

815

          taosArrayRemove(chArray, index);

×

816

          if (taosArrayGetSize(chArray) == 0) {

×

818

            taosArrayDestroy(chArray);

×

819

            int32_t tmpRes = taosHashRemove(pMap, &winRes, sizeof(SWinKey));

×

820

            qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

821

            res = true;

×

822

            qDebug("===stream===retrive pull data over.window %" PRId64, winRes.ts);

×

824

            void* pFinalCh = taosHashGet(pFinalMap, &winRes, sizeof(SWinKey));

×

825

            if (pFinalCh) {

×

826

              int32_t tmpRes = taosHashRemove(pFinalMap, &winRes, sizeof(SWinKey));

×

827

              qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

828

              doDeleteWindow(pOperator, winRes.ts, winRes.groupId);

×

829

              STimeWindow     nextWin = getFinalTimeWindow(winRes.ts, pInterval);

×

830

              SPullWindowInfo pull = {.window = nextWin,

×

831

                                      .groupId = winRes.groupId,

×

832

                                      .calWin.skey = nextWin.skey,

×

833

                                      .calWin.ekey = nextWin.skey};

×

835

              qDebug("===stream===prepare final retrive for delete window:%" PRId64 ",groupId:%" PRId64 ", size:%d",

×

837

              if (IS_MID_INTERVAL_OP(pOperator)) {

×

838

                SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info;

×

840

                void* tmp = taosArrayPush(pInfo->pMidPullDatas, &winRes);

×

841

                if (!tmp) {

×

842

                  code = terrno;

×

843

                  QUERY_CHECK_CODE(code, lino, _end);

×

845

              } else if (savePullWindow(&pull, pPullWins) == TSDB_CODE_SUCCESS) {

×

846

                void* tmp = taosArrayPush(pInfo->pDelWins, &winRes);

×

847

                if (!tmp) {

×

848

                  code = terrno;

×

849

                  QUERY_CHECK_CODE(code, lino, _end);

×

852

                code = addPullWindow(pMap, &winRes, numOfCh);

×

853

                QUERY_CHECK_CODE(code, lino, _end);

×

855

                if (pInfo->destHasPrimaryKey) {

×

856

                  code = tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0);

×

857

                  QUERY_CHECK_CODE(code, lino, _end);

×

859

                qDebug("===stream===prepare final retrive for delete %" PRId64 ", size:%d", winRes.ts, numOfCh);

×

865

      winTs = taosTimeAdd(winTs, pInterval->sliding, pInterval->slidingUnit, pInterval->precision, NULL);

×

868

  if (pBeOver) {

×

869

    *pBeOver = res;

×

872

_end:

×

873

  if (code != TSDB_CODE_SUCCESS) {

×

874

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

876

  return code;

×

879

static int32_t addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, int32_t childId) {

×

880

  int32_t code = TSDB_CODE_SUCCESS;

×

881

  int32_t lino = 0;

×

882

  int32_t size = taosArrayGetSize(wins);

×

883

  for (int32_t i = 0; i < size; i++) {

×

884

    SWinKey*    winKey = taosArrayGet(wins, i);

×

885

    STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval);

×

886

    void*       chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey));

×

887

    if (!chIds) {

×

888

      SPullWindowInfo pull = {

×

889

          .window = nextWin, .groupId = winKey->groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey};

×

891

      if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) {

×

892

        code = addPullWindow(pInfo->pPullDataMap, winKey, pInfo->numOfChild);

×

893

        QUERY_CHECK_CODE(code, lino, _end);

×

895

        if (pInfo->destHasPrimaryKey) {

×

896

          code = tSimpleHashPut(pInfo->pDeletedMap, winKey, sizeof(SWinKey), NULL, 0);

×

897

          QUERY_CHECK_CODE(code, lino, _end);

×

899

        qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, pInfo->numOfChild);

×

902

      SArray* chArray = *(void**)chIds;

×

903

      int32_t index = taosArraySearchIdx(chArray, &childId, compareInt32Val, TD_EQ);

×

904

      qDebug("===stream===check final retrive %" PRId64 ",chid:%d", winKey->ts, index);

×

905

      if (index == -1) {

×

906

        qDebug("===stream===add final retrive %" PRId64, winKey->ts);

×

907

        code = taosHashPut(pInfo->pFinalPullDataMap, winKey, sizeof(SWinKey), NULL, 0);

×

908

        QUERY_CHECK_CODE(code, lino, _end);

×

913

_end:

×

914

  if (code != TSDB_CODE_SUCCESS) {

×

915

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

917

  return code;

×

920

static void clearFunctionContext(SExprSupp* pSup) {

×

921

  for (int32_t i = 0; i < pSup->numOfExprs; i++) {

×

922

    pSup->pCtx[i].saveHandle.currentPage = -1;

×

924

×

926

int32_t getOutputBuf(void* pState, SRowBuffPos* pPos, SResultRow** pResult, SStateStore* pStore) {

×

927

  return pStore->streamStateGetByPos(pState, pPos, (void**)pResult);

×

930

void buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup,

×

932

  int32_t         code = TSDB_CODE_SUCCESS;

×

933

  int32_t         lino = 0;

×

934

  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;

×

935

  SStorageAPI*    pAPI = &pOperator->pTaskInfo->storageAPI;

×

936

  SExprInfo*      pExprInfo = pSup->pExprInfo;

×

937

  int32_t         numOfExprs = pSup->numOfExprs;

×

938

  int32_t*        rowEntryOffset = pSup->rowEntryInfoOffset;

×

939

  SqlFunctionCtx* pCtx = pSup->pCtx;

×

941

  int32_t numOfRows = getNumOfTotalRes(pGroupResInfo);

×

943

  for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) {

×

944

    SRowBuffPos* pPos = *(SRowBuffPos**)taosArrayGet(pGroupResInfo->pRows, i);

×

945

    SResultRow*  pRow = NULL;

×

946

    code = getOutputBuf(pState, pPos, &pRow, &pAPI->stateStore);

×

947

    QUERY_CHECK_CODE(code, lino, _end);

×

948

    uint64_t groupId = ((SWinKey*)pPos->pKey)->groupId;

×

949

    doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset);

×

951

    if (pRow->numOfRows == 0) {

×

952

      pGroupResInfo->index += 1;

×

953

      continue;

×

955

    if (pBlock->info.id.groupId == 0) {

×

956

      pBlock->info.id.groupId = groupId;

×

957

      void*   tbname = NULL;

×

958

      int32_t winCode = TSDB_CODE_SUCCESS;

×

959

      code = pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname,

×

961

      QUERY_CHECK_CODE(code, lino, _end);

×

962

      if (winCode != TSDB_CODE_SUCCESS) {

×

963

        pBlock->info.parTbName[0] = 0;

×

965

        memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);

×

967

      qDebug("%s partName:%s, groupId:%"PRIu64, __FUNCTION__, (char*)tbname, groupId);

×

969

      pAPI->stateStore.streamStateFreeVal(tbname);

×

972

      if (pBlock->info.id.groupId != groupId) {

×

973

        break;

×

977

    if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) {

×

978

      break;

×

980

    pGroupResInfo->index += 1;

×

982

    for (int32_t j = 0; j < numOfExprs; ++j) {

×

983

      int32_t slotId = pExprInfo[j].base.resSchema.slotId;

×

985

      pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset);

×

987

      if (pCtx[j].fpSet.finalize) {

×

988

        int32_t tmpRes = pCtx[j].fpSet.finalize(&pCtx[j], pBlock);

×

989

        if (TAOS_FAILED(tmpRes)) {

×

990

          qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(tmpRes));

×

991

          QUERY_CHECK_CODE(code, lino, _end);

×

993

      } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) {

×

998

        SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId);

×

999

        char*            in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo);

×

1000

        for (int32_t k = 0; k < pRow->numOfRows; ++k) {

×

1001

          code = colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes);

×

1002

          QUERY_CHECK_CODE(code, lino, _end);

×

1007

    if (pSessionKeys) {

×

1008

      SSessionKey key = {.groupId = groupId, .win = pRow->win};

×

1009

      for (int32_t j = 0; j < pRow->numOfRows; ++j) {

×

1010

        const void* px = taosArrayPush(pSessionKeys, &key);

×

1011

        QUERY_CHECK_NULL(px, code, lino, _end, terrno);

×

1015

    pBlock->info.rows += pRow->numOfRows;

×

1018

  pBlock->info.dataLoad = 1;

×

1019

  code = blockDataUpdateTsWindow(pBlock, 0);

×

1020

  QUERY_CHECK_CODE(code, lino, _end);

×

1022

_end:

×

1023

  if (code != TSDB_CODE_SUCCESS) {

×

1024

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

1026

×

1028

void doBuildStreamIntervalResult(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock,

×

1030

  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

×

1032

  pBlock->info.version = pTaskInfo->version;

×

1034

  blockDataCleanup(pBlock);

×

1035

  taosArrayClear(pSessionKeys);

×

1036

  if (!hasRemainResults(pGroupResInfo)) {

×

1037

    return;

×

1041

  pBlock->info.id.groupId = 0;

×

1042

  buildDataBlockFromGroupRes(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo, pSessionKeys);

×

1045

int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,

×

1047

  int32_t startPos = prevPosition + 1;

×

1048

  if (startPos == pDataBlockInfo->rows) {

×

1049

    startPos = -1;

×

1051

    *pNext = getFinalTimeWindow(primaryKeys[startPos], pInterval);

×

1053

  return startPos;

×

1056

static int32_t doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, uint64_t groupId,

×

1058

  int32_t                      code = TSDB_CODE_SUCCESS;

×

1059

  int32_t                      lino = 0;

×

1060

  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info;

×

1061

  pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);

×

1063

  SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo);

×

1064

  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;

×

1065

  SExprSupp*      pSup = &pOperator->exprSupp;

×

1066

  int32_t         numOfOutput = pSup->numOfExprs;

×

1067

  int32_t         step = 1;

×

1068

  TSKEY*          tsCols = NULL;

×

1069

  SRowBuffPos*    pResPos = NULL;

×

1070

  SResultRow*     pResult = NULL;

×

1071

  int32_t         forwardRows = 0;

×

1072

  int32_t         endRowId = pSDataBlock->info.rows - 1;

×

1074

  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);

×

1075

  tsCols = (int64_t*)pColDataInfo->pData;

×

1077

  void*            pPkVal = NULL;

×

1078

  int32_t          pkLen = 0;

×

1079

  SColumnInfoData* pPkColDataInfo = NULL;

×

1080

  if (hasSrcPrimaryKeyCol(&pInfo->basic)) {

×

1081

    pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex);

×

1084

  if (pSDataBlock->info.window.skey != tsCols[0] || pSDataBlock->info.window.ekey != tsCols[endRowId]) {

×

1085

    qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 ",maxKey %" PRId64,

×

1087

    code = blockDataUpdateTsWindow(pSDataBlock, pInfo->primaryTsIndex);

×

1088

    QUERY_CHECK_CODE(code, lino, _end);

×

1091

    if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) {

×

1092

      qError("table uid %" PRIu64 " data block timestamp is out of range! minKey %" PRId64 ",maxKey %" PRId64,

×

1097

  int32_t     startPos = 0;

×

1098

  TSKEY       ts = getStartTsKey(&pSDataBlock->info.window, tsCols);

×

1099

  STimeWindow nextWin = {0};

×

1100

  if (IS_FINAL_INTERVAL_OP(pOperator)) {

×

1101

    nextWin = getFinalTimeWindow(ts, &pInfo->interval);

×

1103

    nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC);

×

1105

  while (1) {

×

1106

    bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);

×

1107

    if (hasSrcPrimaryKeyCol(&pInfo->basic) && !IS_FINAL_INTERVAL_OP(pOperator) && pInfo->ignoreExpiredData &&

×

1108

        pSDataBlock->info.type != STREAM_PULL_DATA) {

×

1109

      pPkVal = colDataGetData(pPkColDataInfo, startPos);

×

1110

      pkLen = colDataGetRowLength(pPkColDataInfo, startPos);

×

1113

    if ((!IS_FINAL_INTERVAL_OP(pOperator) && pInfo->ignoreExpiredData && pSDataBlock->info.type != STREAM_PULL_DATA &&

×

1114

         checkExpiredData(&pInfo->stateStore, pInfo->pUpdateInfo, &pInfo->twAggSup, pSDataBlock->info.id.uid,

×

1115

                          nextWin.ekey, pPkVal, pkLen)) ||

×

1116

        !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) {

×

1117

      startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);

×

1118

      if (startPos < 0) {

×

1119

        break;

×

1121

      qDebug("===stream===ignore expired data, window end ts:%" PRId64 ", maxts - wartermak:%" PRId64, nextWin.ekey,

×

1123

      continue;

×

1126

    if (IS_FINAL_INTERVAL_OP(pOperator) && pInfo->numOfChild > 0) {

×

1127

      bool    ignore = true;

×

1128

      SWinKey winRes = {

×

1129

          .ts = nextWin.skey,

×

1132

      void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey));

×

1133

      if (isDeletedStreamWindow(&nextWin, groupId, pInfo->pState, &pInfo->twAggSup, &pInfo->stateStore) && isClosed &&

×

1134

          !chIds) {

×

1135

        SPullWindowInfo pull = {

×

1136

            .window = nextWin, .groupId = groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey};

×

1138

        if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) {

×

1139

          code = addPullWindow(pInfo->pPullDataMap, &winRes, pInfo->numOfChild);

×

1140

          QUERY_CHECK_CODE(code, lino, _end);

×

1142

          if (pInfo->destHasPrimaryKey) {

×

1143

            code = tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0);

×

1144

            QUERY_CHECK_CODE(code, lino, _end);

×

1148

        int32_t index = -1;

×

1149

        SArray* chArray = NULL;

×

1150

        int32_t chId = 0;

×

1151

        if (chIds) {

×

1152

          chArray = *(void**)chIds;

×

1153

          chId = getChildIndex(pSDataBlock);

×

1154

          index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);

×

1156

        if (index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA) {

×

1157

          ignore = false;

×

1161

      if (ignore) {

×

1162

        startPos = getNextQualifiedFinalWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, startPos);

×

1163

        if (startPos < 0) {

×

1164

          break;

×

1166

        continue;

×

1170

    int32_t winCode = TSDB_CODE_SUCCESS;

×

1171

    code = setIntervalOutputBuf(pInfo->pState, &nextWin, &pResPos, groupId, pSup->pCtx, numOfOutput,

×

1173

    QUERY_CHECK_CODE(code, lino, _end);

×

1175

    if (winCode != TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator) &&

×

1176

        BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {

×

1177

      SSessionKey key = {.win = nextWin, .groupId = groupId};

×

1178

      code = addIntervalAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &key, &pInfo->basic.notifyEventSup,

×

1180

      QUERY_CHECK_CODE(code, lino, _end);

×

1183

    pResult = (SResultRow*)pResPos->pRowBuff;

×

1185

    if (IS_FINAL_INTERVAL_OP(pOperator)) {

×

1186

      forwardRows = 1;

×

1188

      forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey,

×

1192

    SWinKey key = {

×

1193

        .ts = pResult->win.skey,

×

1197

    if (pInfo->destHasPrimaryKey && winCode == TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator)) {

×

1198

      code = tSimpleHashPut(pDeletedMap, &key, sizeof(SWinKey), NULL, 0);

×

1199

      QUERY_CHECK_CODE(code, lino, _end);

×

1202

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) {

×

1203

      code = saveWinResult(&key, pResPos, pUpdatedMap);

×

1204

      QUERY_CHECK_CODE(code, lino, _end);

×

1207

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

1208

      pResPos->beUpdated = true;

×

1209

      code = tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pResPos, POINTER_BYTES);

×

1210

      QUERY_CHECK_CODE(code, lino, _end);

×

1213

    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, 1);

×

1214

    code = applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos,

×

1215

                                           forwardRows, pSDataBlock->info.rows, numOfOutput);

×

1216

    QUERY_CHECK_CODE(code, lino, _end);

×

1217

    key.ts = nextWin.skey;

×

1219

    if (pInfo->delKey.ts > key.ts) {

×

1220

      pInfo->delKey = key;

×

1222

    int32_t prevEndPos = (forwardRows - 1) * step + startPos;

×

1223

    if (IS_FINAL_INTERVAL_OP(pOperator)) {

×

1224

      startPos = getNextQualifiedFinalWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos);

×

1227

          getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC);

×

1229

    if (startPos < 0) {

×

1230

      break;

×

1233

_end:

×

1234

  if (code != TSDB_CODE_SUCCESS) {

×

1235

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

1237

  return code;

×

1240

int winPosCmprImpl(const void* pKey1, const void* pKey2) {

×

1241

  SRowBuffPos* pos1 = *(SRowBuffPos**)pKey1;

×

1242

  SRowBuffPos* pos2 = *(SRowBuffPos**)pKey2;

×

1243

  SWinKey*     pWin1 = (SWinKey*)pos1->pKey;

×

1244

  SWinKey*     pWin2 = (SWinKey*)pos2->pKey;

×

1246

  if (pWin1->groupId > pWin2->groupId) {

×

1247

    return 1;

×

1248

  } else if (pWin1->groupId < pWin2->groupId) {

×

1249

    return -1;

×

1252

  if (pWin1->ts > pWin2->ts) {

×

1253

    return 1;

×

1254

  } else if (pWin1->ts < pWin2->ts) {

×

1255

    return -1;

×

1258

  return 0;

×

1261

static void resetUnCloseWinInfo(SSHashObj* winMap) {

×

1262

  void*   pIte = NULL;

×

1263

  int32_t iter = 0;

×

1264

  while ((pIte = tSimpleHashIterate(winMap, pIte, &iter)) != NULL) {

×

1265

    SRowBuffPos* pPos = *(SRowBuffPos**)pIte;

×

1266

    pPos->beUsed = true;

×

1268

×

1270

int32_t encodeSWinKey(void** buf, SWinKey* key) {

×

1271

  int32_t tlen = 0;

×

1272

  tlen += taosEncodeFixedI64(buf, key->ts);

×

1273

  tlen += taosEncodeFixedU64(buf, key->groupId);

×

1274

  tlen += taosEncodeFixedI32(buf, key->numInGroup);

×

1275

  return tlen;

×

1278

void* decodeSWinKey(void* buf, SWinKey* key) {

×

1279

  buf = taosDecodeFixedI64(buf, &key->ts);

×

1280

  buf = taosDecodeFixedU64(buf, &key->groupId);

×

1281

  buf = taosDecodeFixedI32(buf, &key->numInGroup);

×

1282

  return buf;

×

1285

int32_t encodeSTimeWindowAggSupp(void** buf, STimeWindowAggSupp* pTwAggSup) {

×

1286

  int32_t tlen = 0;

×

1287

  tlen += taosEncodeFixedI64(buf, pTwAggSup->minTs);

×

1288

  tlen += taosEncodeFixedI64(buf, pTwAggSup->maxTs);

×

1289

  return tlen;

×

1292

void* decodeSTimeWindowAggSupp(void* buf, STimeWindowAggSupp* pTwAggSup) {

×

1293

  buf = taosDecodeFixedI64(buf, &pTwAggSup->minTs);

×

1294

  buf = taosDecodeFixedI64(buf, &pTwAggSup->maxTs);

×

1295

  return buf;

×

1298

int32_t encodeSTimeWindow(void** buf, STimeWindow* pWin) {

×

1299

  int32_t tlen = 0;

×

1300

  tlen += taosEncodeFixedI64(buf, pWin->skey);

×

1301

  tlen += taosEncodeFixedI64(buf, pWin->ekey);

×

1302

  return tlen;

×

1305

void* decodeSTimeWindow(void* buf, STimeWindow* pWin) {

×

1306

  buf = taosDecodeFixedI64(buf, &pWin->skey);

×

1307

  buf = taosDecodeFixedI64(buf, &pWin->ekey);

×

1308

  return buf;

×

1311

int32_t encodeSPullWindowInfo(void** buf, SPullWindowInfo* pPullInfo) {

×

1312

  int32_t tlen = 0;

×

1313

  tlen += encodeSTimeWindow(buf, &pPullInfo->calWin);

×

1314

  tlen += taosEncodeFixedU64(buf, pPullInfo->groupId);

×

1315

  tlen += encodeSTimeWindow(buf, &pPullInfo->window);

×

1316

  return tlen;

×

1319

void* decodeSPullWindowInfo(void* buf, SPullWindowInfo* pPullInfo) {

×

1320

  buf = decodeSTimeWindow(buf, &pPullInfo->calWin);

×

1321

  buf = taosDecodeFixedU64(buf, &pPullInfo->groupId);

×

1322

  buf = decodeSTimeWindow(buf, &pPullInfo->window);

×

1323

  return buf;

×

1326

int32_t encodeSPullWindowInfoArray(void** buf, SArray* pPullInfos) {

×

1327

  int32_t tlen = 0;

×

1328

  int32_t size = taosArrayGetSize(pPullInfos);

×

1329

  tlen += taosEncodeFixedI32(buf, size);

×

1330

  for (int32_t i = 0; i < size; i++) {

×

1331

    void* pItem = taosArrayGet(pPullInfos, i);

×

1332

    tlen += encodeSPullWindowInfo(buf, pItem);

×

1334

  return tlen;

×

1337

int32_t decodeSPullWindowInfoArray(void* buf, SArray* pPullInfos, void** ppBuf) {

×

1338

  int32_t code = TSDB_CODE_SUCCESS;

×

1339

  int32_t lino = 0;

×

1340

  int32_t size = 0;

×

1341

  buf = taosDecodeFixedI32(buf, &size);

×

1342

  for (int32_t i = 0; i < size; i++) {

×

1343

    SPullWindowInfo item = {0};

×

1344

    buf = decodeSPullWindowInfo(buf, &item);

×

1345

    void* tmp = taosArrayPush(pPullInfos, &item);

×

1346

    if (!tmp) {

×

1347

      code = terrno;

×

1348

      QUERY_CHECK_CODE(code, lino, _end);

×

1351

  (*ppBuf) = buf;

×

1353

_end:

×

1354

  if (code != TSDB_CODE_SUCCESS) {

×

1355

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

1357

  return code;

×

1360

int32_t doStreamIntervalEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOperator) {

×

1361

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1362

  if (!pInfo) {

×

1363

    return 0;

×

1366

  void* pData = (buf == NULL) ? NULL : *buf;

×

1369

  int32_t tlen = 0;

×

1370

  int32_t mapSize = tSimpleHashGetSize(pInfo->aggSup.pResultRowHashTable);

×

1371

  tlen += taosEncodeFixedI32(buf, mapSize);

×

1372

  void*   pIte = NULL;

×

1373

  size_t  keyLen = 0;

×

1374

  int32_t iter = 0;

×

1375

  while ((pIte = tSimpleHashIterate(pInfo->aggSup.pResultRowHashTable, pIte, &iter)) != NULL) {

×

1376

    void* key = tSimpleHashGetKey(pIte, &keyLen);

×

1377

    tlen += encodeSWinKey(buf, key);

×

1381

  tlen += encodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);

×

1384

  int32_t size = taosHashGetSize(pInfo->pPullDataMap);

×

1385

  tlen += taosEncodeFixedI32(buf, size);

×

1386

  pIte = NULL;

×

1387

  keyLen = 0;

×

1388

  while ((pIte = taosHashIterate(pInfo->pPullDataMap, pIte)) != NULL) {

×

1389

    void* key = taosHashGetKey(pIte, &keyLen);

×

1390

    tlen += encodeSWinKey(buf, key);

×

1391

    SArray* pArray = *(SArray**)pIte;

×

1392

    int32_t chSize = taosArrayGetSize(pArray);

×

1393

    tlen += taosEncodeFixedI32(buf, chSize);

×

1394

    for (int32_t i = 0; i < chSize; i++) {

×

1395

      void* pChItem = taosArrayGet(pArray, i);

×

1396

      tlen += taosEncodeFixedI32(buf, *(int32_t*)pChItem);

×

1401

  tlen += encodeSPullWindowInfoArray(buf, pInfo->pPullWins);

×

1404

  tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);

×

1407

  tlen += encodeStreamBasicInfo(buf, &pInfo->basic);

×

1410

  if (buf) {

×

1411

    uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));

×

1412

    tlen += taosEncodeFixedU32(buf, cksum);

×

1414

    tlen += sizeof(uint32_t);

×

1417

  return tlen;

×

1420

void doStreamIntervalDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator) {

×

1421

  int32_t                      code = TSDB_CODE_SUCCESS;

×

1422

  int32_t                      lino = 0;

×

1423

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1424

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

1425

  void*                        pDataEnd = POINTER_SHIFT(buf, len);

×

1426

  if (!pInfo) {

×

1427

    code = TSDB_CODE_FAILED;

×

1428

    QUERY_CHECK_CODE(code, lino, _end);

×

1432

  int32_t dataLen = len - sizeof(uint32_t);

×

1433

  void*   pCksum = POINTER_SHIFT(buf, dataLen);

×

1434

  if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) {

×

1435

    code = TSDB_CODE_FAILED;

×

1436

    QUERY_CHECK_CODE(code, lino, _end);

×

1438

  pDataEnd = pCksum;

×

1441

  int32_t mapSize = 0;

×

1442

  buf = taosDecodeFixedI32(buf, &mapSize);

×

1443

  for (int32_t i = 0; i < mapSize; i++) {

×

1444

    SWinKey key = {0};

×

1445

    buf = decodeSWinKey(buf, &key);

×

1446

    SRowBuffPos* pPos = NULL;

×

1447

    int32_t      resSize = pInfo->aggSup.resultRowSize;

×

1448

    int32_t      winCode = TSDB_CODE_SUCCESS;

×

1449

    code = pInfo->stateStore.streamStateAddIfNotExist(pInfo->pState, &key, (void**)&pPos, &resSize, &winCode);

×

1450

    QUERY_CHECK_CODE(code, lino, _end);

×

1451

    QUERY_CHECK_CONDITION((winCode == TSDB_CODE_SUCCESS), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);

×

1453

    code = tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pPos, POINTER_BYTES);

×

1454

    QUERY_CHECK_CODE(code, lino, _end);

×

1458

  buf = decodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);

×

1461

  int32_t size = 0;

×

1462

  buf = taosDecodeFixedI32(buf, &size);

×

1463

  for (int32_t i = 0; i < size; i++) {

×

1464

    SWinKey key = {0};

×

1465

    SArray* pArray = taosArrayInit(0, sizeof(int32_t));

×

1466

    if (!pArray) {

×

1467

      code = terrno;

×

1468

      QUERY_CHECK_CODE(code, lino, _end);

×

1471

    buf = decodeSWinKey(buf, &key);

×

1472

    int32_t chSize = 0;

×

1473

    buf = taosDecodeFixedI32(buf, &chSize);

×

1474

    for (int32_t i = 0; i < chSize; i++) {

×

1475

      int32_t chId = 0;

×

1476

      buf = taosDecodeFixedI32(buf, &chId);

×

1477

      void* tmp = taosArrayPush(pArray, &chId);

×

1478

      if (!tmp) {

×

1479

        code = terrno;

×

1480

        QUERY_CHECK_CODE(code, lino, _end);

×

1483

    code = taosHashPut(pInfo->pPullDataMap, &key, sizeof(SWinKey), &pArray, POINTER_BYTES);

×

1484

    QUERY_CHECK_CODE(code, lino, _end);

×

1488

  code = decodeSPullWindowInfoArray(buf, pInfo->pPullWins, &buf);

×

1489

  QUERY_CHECK_CODE(code, lino, _end);

×

1492

  buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);

×

1495

  if (buf < pDataEnd) {

×

1496

    code = decodeStreamBasicInfo(&buf, &pInfo->basic);

×

1497

    QUERY_CHECK_CODE(code, lino, _end);

×

1500

_end:

×

1501

  if (code != TSDB_CODE_SUCCESS) {

×

1502

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

1504

×

1506

void doStreamIntervalSaveCheckpoint(SOperatorInfo* pOperator) {

×

1507

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1508

  if (needSaveStreamOperatorInfo(&pInfo->basic)) {

×

1509

    int32_t len = doStreamIntervalEncodeOpState(NULL, 0, pOperator);

×

1510

    void*   buf = taosMemoryCalloc(1, len);

×

1511

    if (!buf) {

×

1512

      qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno));

×

1513

      return;

×

1515

    void* pBuf = buf;

×

1516

    len = doStreamIntervalEncodeOpState(&pBuf, len, pOperator);

×

1517

    pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_INTERVAL_OP_CHECKPOINT_NAME,

×

1519

    taosMemoryFree(buf);

×

1520

    saveStreamOperatorStateComplete(&pInfo->basic);

×

1524

int32_t copyIntervalDeleteKey(SSHashObj* pMap, SArray* pWins) {

×

1525

  int32_t code = TSDB_CODE_SUCCESS;

×

1526

  int32_t lino = 0;

×

1527

  void*   pIte = NULL;

×

1528

  int32_t iter = 0;

×

1529

  while ((pIte = tSimpleHashIterate(pMap, pIte, &iter)) != NULL) {

×

1530

    void* pKey = tSimpleHashGetKey(pIte, NULL);

×

1531

    void* tmp = taosArrayPush(pWins, pKey);

×

1532

    if (!tmp) {

×

1533

      code = terrno;

×

1534

      QUERY_CHECK_CODE(code, lino, _end);

×

1537

  tSimpleHashClear(pMap);

×

1539

_end:

×

1540

  if (code != TSDB_CODE_SUCCESS) {

×

1541

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

1543

  return code;

×

1546

static int32_t buildIntervalResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

1547

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1548

  int32_t                      code = TSDB_CODE_SUCCESS;

×

1549

  int32_t                      lino = 0;

×

1550

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

1551

  uint16_t                     opType = pOperator->operatorType;

×

1552

  SStreamNotifyEventSupp*      pNotifySup = &pInfo->basic.notifyEventSup;

×

1553

  STaskNotifyEventStat*        pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;

×

1554

  bool                         addNotifyEvent = false;

×

1557

  if (isTaskKilled(pTaskInfo)) {

×

1558

    (*ppRes) = NULL;

×

1559

    return code;

×

1562

  addNotifyEvent = IS_NORMAL_INTERVAL_OP(pOperator) &&

×

1563

                   BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);

×

1564

  if (IS_FINAL_INTERVAL_OP(pOperator)) {

×

1565

    doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);

×

1566

    if (pInfo->pPullDataRes->info.rows != 0) {

×

1568

      printDataBlock(pInfo->pPullDataRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));

×

1569

      (*ppRes) = pInfo->pPullDataRes;

×

1570

      return code;

×

1574

  doBuildDeleteResult(pTaskInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);

×

1575

  if (pInfo->pDelRes->info.rows != 0) {

×

1577

    printDataBlock(pInfo->pDelRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));

×

1578

    if (addNotifyEvent) {

×

1579

      code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);

×

1580

      QUERY_CHECK_CODE(code, lino, _end);

×

1582

    (*ppRes) = pInfo->pDelRes;

×

1583

    return code;

×

1586

  doBuildStreamIntervalResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo,

×

1588

  if (pInfo->binfo.pRes->info.rows != 0) {

×

1589

    printDataBlock(pInfo->binfo.pRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));

×

1590

    if (addNotifyEvent) {

×

1591

      code = addAggResultNotifyEvent(pInfo->binfo.pRes, pNotifySup->pSessionKeys,

×

1592

                                     pTaskInfo->streamInfo.notifyResultSchema, pNotifySup, pNotifyEventStat);

×

1593

      QUERY_CHECK_CODE(code, lino, _end);

×

1595

    (*ppRes) = pInfo->binfo.pRes;

×

1596

    return code;

×

1599

  code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);

×

1600

  QUERY_CHECK_CODE(code, lino, _end);

×

1601

  if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {

×

1602

    printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

1603

    (*ppRes) = pNotifySup->pEventBlock;

×

1604

    return code;

×

1607

  code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);

×

1608

  QUERY_CHECK_CODE(code, lino, _end);

×

1610

_end:

×

1611

  if (code != TSDB_CODE_SUCCESS) {

×

1612

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

1614

  (*ppRes) = NULL;

×

1615

  return code;

×

1618

int32_t copyUpdateResult(SSHashObj** ppWinUpdated, SArray* pUpdated, __compar_fn_t compar) {

×

1619

  int32_t code = TSDB_CODE_SUCCESS;

×

1620

  int32_t lino = 0;

×

1621

  void*   pIte = NULL;

×

1622

  int32_t iter = 0;

×

1623

  while ((pIte = tSimpleHashIterate(*ppWinUpdated, pIte, &iter)) != NULL) {

×

1624

    void* tmp = taosArrayPush(pUpdated, pIte);

×

1625

    if (!tmp) {

×

1626

      code = terrno;

×

1627

      QUERY_CHECK_CODE(code, lino, _end);

×

1630

  taosArraySort(pUpdated, compar);

×

1631

  tSimpleHashCleanup(*ppWinUpdated);

×

1632

  *ppWinUpdated = NULL;

×

1634

_end:

×

1635

  if (code != TSDB_CODE_SUCCESS) {

×

1636

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

1638

  return code;

×

1641

static int32_t doStreamFinalIntervalAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

1642

  int32_t                      code = TSDB_CODE_SUCCESS;

×

1643

  int32_t                      lino = 0;

×

1644

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1645

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

1646

  SStorageAPI*                 pAPI = &pOperator->pTaskInfo->storageAPI;

×

1648

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

1649

  SExprSupp*     pSup = &pOperator->exprSupp;

×

1651

  qDebug("stask:%s  %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);

×

1653

  if (pOperator->status == OP_EXEC_DONE) {

×

1654

    (*ppRes) = NULL;

×

1655

    return code;

×

1656

  } else if (pOperator->status == OP_RES_TO_RETURN) {

×

1657

    SSDataBlock* resBlock = NULL;

×

1658

    code = buildIntervalResult(pOperator, &resBlock);

×

1659

    QUERY_CHECK_CODE(code, lino, _end);

×

1660

    if (resBlock != NULL) {

×

1661

      (*ppRes) = resBlock;

×

1662

      return code;

×

1665

    if (pInfo->recvGetAll) {

×

1666

      pInfo->recvGetAll = false;

×

1667

      resetUnCloseWinInfo(pInfo->aggSup.pResultRowHashTable);

×

1670

    if (pInfo->reCkBlock) {

×

1671

      pInfo->reCkBlock = false;

×

1672

      printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

1673

      (*ppRes) = pInfo->pCheckpointRes;

×

1674

      return code;

×

1677

    setStreamOperatorCompleted(pOperator);

×

1678

    if (!IS_FINAL_INTERVAL_OP(pOperator)) {

×

1679

      clearFunctionContext(&pOperator->exprSupp);

×

1681

      clearStreamIntervalOperator(pInfo);

×

1682

      qDebug("===stream===clear semi operator");

×

1684

    (*ppRes) = NULL;

×

1685

    return code;

×

1687

    if (!IS_FINAL_INTERVAL_OP(pOperator)) {

×

1688

      SSDataBlock* resBlock = NULL;

×

1689

      code = buildIntervalResult(pOperator, &resBlock);

×

1690

      QUERY_CHECK_CODE(code, lino, _end);

×

1691

      if (resBlock != NULL) {

×

1692

        (*ppRes) = resBlock;

×

1693

        return code;

×

1696

      if (pInfo->recvRetrive) {

×

1697

        pInfo->recvRetrive = false;

×

1698

        printDataBlock(pInfo->pMidRetriveRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

1699

        (*ppRes) = pInfo->pMidRetriveRes;

×

1700

        return code;

×

1705

  if (!pInfo->pUpdated) {

×

1706

    pInfo->pUpdated = taosArrayInit(4096, POINTER_BYTES);

×

1707

    QUERY_CHECK_NULL(pInfo->pUpdated, code, lino, _end, terrno);

×

1709

  if (!pInfo->pUpdatedMap) {

×

1710

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

1711

    pInfo->pUpdatedMap = tSimpleHashInit(4096, hashFn);

×

1712

    QUERY_CHECK_NULL(pInfo->pUpdatedMap, code, lino, _end, terrno);

×

1715

  while (1) {

×

1716

    if (isTaskKilled(pTaskInfo)) {

×

1717

      qInfo("===stream=== %s task is killed, code %s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code));

×

1718

      (*ppRes) = NULL;

×

1719

      return code;

×

1722

    SSDataBlock* pBlock = NULL;

×

1723

    code = downstream->fpSet.getNextFn(downstream, &pBlock);

×

1724

    QUERY_CHECK_CODE(code, lino, _end);

×

1726

    if (pBlock == NULL) {

×

1727

      pOperator->status = OP_RES_TO_RETURN;

×

1728

      qDebug("===stream===return data:%s. recv datablock num:%" PRIu64, getStreamOpName(pOperator->operatorType),

×

1730

      pInfo->numOfDatapack = 0;

×

1731

      break;

×

1734

    pInfo->numOfDatapack++;

×

1735

    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));

×

1736

    setStreamOperatorState(&pInfo->basic, pBlock->info.type);

×

1738

    if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) {

×

1739

      pInfo->binfo.pRes->info.type = pBlock->info.type;

×

1740

    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||

×

1741

               pBlock->info.type == STREAM_CLEAR) {

×

1742

      SArray* delWins = taosArrayInit(8, sizeof(SWinKey));

×

1743

      QUERY_CHECK_NULL(delWins, code, lino, _end, terrno);

×

1744

      SHashObj* finalMap = IS_FINAL_INTERVAL_OP(pOperator) ? pInfo->pFinalPullDataMap : NULL;

×

1745

      code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, finalMap);

×

1746

      QUERY_CHECK_CODE(code, lino, _end);

×

1748

      if (IS_FINAL_INTERVAL_OP(pOperator)) {

×

1749

        int32_t chId = getChildIndex(pBlock);

×

1750

        code = addRetriveWindow(delWins, pInfo, chId);

×

1751

        QUERY_CHECK_CODE(code, lino, _end);

×

1753

        if (pBlock->info.type != STREAM_CLEAR) {

×

1754

          void* tmp = taosArrayAddAll(pInfo->pDelWins, delWins);

×

1755

          if (!tmp && taosArrayGetSize(delWins) > 0) {

×

1756

            code = TSDB_CODE_OUT_OF_MEMORY;

×

1757

            QUERY_CHECK_CODE(code, lino, _end);

×

1760

        taosArrayDestroy(delWins);

×

1761

        continue;

×

1763

      removeResults(delWins, pInfo->pUpdatedMap);

×

1764

      void* tmp = taosArrayAddAll(pInfo->pDelWins, delWins);

×

1765

      if (!tmp && taosArrayGetSize(delWins) > 0) {

×

1766

        code = TSDB_CODE_OUT_OF_MEMORY;

×

1767

        QUERY_CHECK_CODE(code, lino, _end);

×

1769

      taosArrayDestroy(delWins);

×

1771

      doBuildDeleteResult(pTaskInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);

×

1772

      if (pInfo->pDelRes->info.rows != 0) {

×

1774

        printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

1775

        if (pBlock->info.type == STREAM_CLEAR) {

×

1776

          pInfo->pDelRes->info.type = STREAM_CLEAR;

×

1778

          pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;

×

1780

        (*ppRes) = pInfo->pDelRes;

×

1781

        return code;

×

1784

      break;

×

1785

    } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_INTERVAL_OP(pOperator)) {

×

1786

      pInfo->recvGetAll = true;

×

1787

      code = getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap);

×

1788

      QUERY_CHECK_CODE(code, lino, _end);

×

1789

      continue;

×

1790

    } else if (pBlock->info.type == STREAM_RETRIEVE) {

×

1791

      if (!IS_FINAL_INTERVAL_OP(pOperator)) {

×

1792

        pInfo->recvRetrive = true;

×

1793

        code = copyDataBlock(pInfo->pMidRetriveRes, pBlock);

×

1794

        QUERY_CHECK_CODE(code, lino, _end);

×

1796

        pInfo->pMidRetriveRes->info.type = STREAM_MID_RETRIEVE;

×

1797

        code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, NULL, pInfo->pUpdatedMap, NULL);

×

1798

        QUERY_CHECK_CODE(code, lino, _end);

×

1799

        break;

×

1801

      continue;

×

1802

    } else if (pBlock->info.type == STREAM_PULL_OVER && IS_FINAL_INTERVAL_OP(pOperator)) {

×

1803

      code = processPullOver(pBlock, pInfo->pPullDataMap, pInfo->pFinalPullDataMap, &pInfo->interval, pInfo->pPullWins,

×

1805

      QUERY_CHECK_CODE(code, lino, _end);

×

1806

      continue;

×

1807

    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {

×

1808

      (*ppRes) = pBlock;

×

1809

      return code;

×

1810

    } else if (pBlock->info.type == STREAM_CHECKPOINT) {

×

1811

      pAPI->stateStore.streamStateCommit(pInfo->pState);

×

1812

      doStreamIntervalSaveCheckpoint(pOperator);

×

1813

      code = copyDataBlock(pInfo->pCheckpointRes, pBlock);

×

1814

      QUERY_CHECK_CODE(code, lino, _end);

×

1816

      continue;

×

1817

    } else if (IS_FINAL_INTERVAL_OP(pOperator) && pBlock->info.type == STREAM_MID_RETRIEVE) {

×

1818

      continue;

×

1820

      if (pBlock->info.type != STREAM_INVALID) {

×

1821

        code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;

×

1822

        QUERY_CHECK_CODE(code, lino, _end);

×

1826

    if (pInfo->scalarSupp.pExprInfo != NULL) {

×

1827

      SExprSupp* pExprSup = &pInfo->scalarSupp;

×

1828

      code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);

×

1829

      QUERY_CHECK_CODE(code, lino, _end);

×

1831

    code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

1832

    QUERY_CHECK_CODE(code, lino, _end);

×

1834

    code = doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pInfo->pUpdatedMap, pInfo->pDeletedMap);

×

1835

    if (code == TSDB_CODE_STREAM_INTERNAL_ERROR) {

×

1836

      code = TSDB_CODE_SUCCESS;

×

1837

      pOperator->status = OP_RES_TO_RETURN;

×

1838

      break;

×

1840

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);

×

1841

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.watermark);

×

1842

    pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey);

×

1845

  if (IS_FINAL_INTERVAL_OP(pOperator) && !pInfo->destHasPrimaryKey) {

×

1846

    removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins);

×

1848

  if (IS_FINAL_INTERVAL_OP(pOperator)) {

×

1849

    code = closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval,

×

1851

    QUERY_CHECK_CODE(code, lino, _end);

×

1853

    if (pInfo->destHasPrimaryKey) {

×

1854

      code = copyIntervalDeleteKey(pInfo->pDeletedMap, pInfo->pDelWins);

×

1855

      QUERY_CHECK_CODE(code, lino, _end);

×

1858

  pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs;

×

1860

  code = copyUpdateResult(&pInfo->pUpdatedMap, pInfo->pUpdated, winPosCmprImpl);

×

1861

  QUERY_CHECK_CODE(code, lino, _end);

×

1863

  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);

×

1864

  pInfo->pUpdated = NULL;

×

1865

  code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);

×

1866

  QUERY_CHECK_CODE(code, lino, _end);

×

1868

  SSDataBlock* resBlock = NULL;

×

1869

  code = buildIntervalResult(pOperator, &resBlock);

×

1870

  QUERY_CHECK_CODE(code, lino, _end);

×

1871

  if (resBlock != NULL) {

×

1872

    (*ppRes) = resBlock;

×

1873

    return code;

×

1876

  if (pInfo->recvRetrive) {

×

1877

    pInfo->recvRetrive = false;

×

1878

    printDataBlock(pInfo->pMidRetriveRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

1879

    (*ppRes) = pInfo->pMidRetriveRes;

×

1880

    return code;

×

1883

_end:

×

1884

  if (code != TSDB_CODE_SUCCESS) {

×

1885

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

1886

    pTaskInfo->code = code;

×

1887

    T_LONG_JMP(pTaskInfo->env, code);

×

1889

  setStreamOperatorCompleted(pOperator);

×

1890

  (*ppRes) = NULL;

×

1891

  return code;

×

1894

int64_t getDeleteMark(SWindowPhysiNode* pWinPhyNode, int64_t interval) {

×

1895

  if (pWinPhyNode->deleteMark <= 0) {

×

1896

    return DEAULT_DELETE_MARK;

×

1898

  int64_t deleteMark = TMAX(pWinPhyNode->deleteMark, pWinPhyNode->watermark);

×

1899

  deleteMark = TMAX(deleteMark, interval);

×

1900

  return deleteMark;

×

1903

int64_t getDeleteMarkFromOption(SStreamNodeOption* pOption) {

×

1904

  if (pOption->deleteMark <= 0) {

×

1905

    return DEAULT_DELETE_MARK;

×

1907

  int64_t deleteMark = TMAX(pOption->deleteMark, pOption->watermark);

×

1908

  return deleteMark;

×

1911

TSKEY compareTs(void* pKey) {

×

1912

  SWinKey* pWinKey = (SWinKey*)pKey;

×

1913

  return pWinKey->ts;

×

1916

static int32_t getSelectivityBufSize(SqlFunctionCtx* pCtx) {

×

1917

  if (pCtx->subsidiaries.rowLen == 0) {

×

1918

    int32_t rowLen = 0;

×

1919

    for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) {

×

1920

      SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j];

×

1921

      rowLen += pc->pExpr->base.resSchema.bytes;

×

1924

    return rowLen + pCtx->subsidiaries.num * sizeof(bool);

×

1926

    return pCtx->subsidiaries.rowLen;

×

1930

static int32_t getMaxFunResSize(SExprSupp* pSup, int32_t numOfCols) {

×

1931

  int32_t size = 0;

×

1932

  for (int32_t i = 0; i < numOfCols; ++i) {

×

1933

    int32_t resSize = getSelectivityBufSize(pSup->pCtx + i);

×

1934

    size = TMAX(size, resSize);

×

1936

  return size;

×

1939

static void streamIntervalReleaseState(SOperatorInfo* pOperator) {

×

1940

  if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL &&

×

1941

      pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {

×

1942

    SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1943

    int32_t                      resSize = sizeof(TSKEY);

×

1944

    pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME,

×

1945

                                          strlen(STREAM_INTERVAL_OP_STATE_NAME), &pInfo->twAggSup.maxTs, resSize);

×

1947

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1948

  SStorageAPI*                 pAPI = &pOperator->pTaskInfo->storageAPI;

×

1949

  pAPI->stateStore.streamStateCommit(pInfo->pState);

×

1950

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

1951

  if (downstream->fpSet.releaseStreamStateFn) {

×

1952

    downstream->fpSet.releaseStreamStateFn(downstream);

×

1954

×

1956

void streamIntervalReloadState(SOperatorInfo* pOperator) {

×

1957

  int32_t                      code = TSDB_CODE_SUCCESS;

×

1958

  int32_t                      lino = 0;

×

1959

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1960

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

1961

  if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL &&

×

1962

      pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {

×

1963

    int32_t size = 0;

×

1964

    void*   pBuf = NULL;

×

1965

    code = pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME,

×

1967

    QUERY_CHECK_CODE(code, lino, _end);

×

1969

    TSKEY ts = *(TSKEY*)pBuf;

×

1970

    taosMemoryFree(pBuf);

×

1971

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);

×

1972

    pInfo->stateStore.streamStateReloadInfo(pInfo->pState, ts);

×

1974

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

1975

  if (downstream->fpSet.reloadStreamStateFn) {

×

1976

    downstream->fpSet.reloadStreamStateFn(downstream);

×

1978

  reloadFromDownStream(downstream, pInfo);

×

1980

_end:

×

1981

  if (code != TSDB_CODE_SUCCESS) {

×

1982

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

1984

×

1986

int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,

×

1988

  QRY_PARAM_CHECK(pOptrInfo);

×

1990

  int32_t                      code = TSDB_CODE_SUCCESS;

×

1991

  int32_t                      lino = 0;

×

1992

  SIntervalPhysiNode*          pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;

×

1993

  SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo));

×

1994

  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));

×

1995

  if (pInfo == NULL || pOperator == NULL) {

×

1996

    code = terrno;

×

1997

    QUERY_CHECK_CODE(code, lino, _error);

×

2000

  pOperator->exprSupp.hasWindowOrGroup = true;

×

2001

  pOperator->pTaskInfo = pTaskInfo;

×

2002

  SStorageAPI* pAPI = &pTaskInfo->storageAPI;

×

2004

  pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval,

×

2005

                                .sliding = pIntervalPhyNode->sliding,

×

2006

                                .intervalUnit = pIntervalPhyNode->intervalUnit,

×

2007

                                .slidingUnit = pIntervalPhyNode->slidingUnit,

×

2008

                                .offset = pIntervalPhyNode->offset,

×

2009

                                .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision,

×

2010

                                .timeRange = pIntervalPhyNode->timeRange};

×

2011

  calcIntervalAutoOffset(&pInfo->interval);

×

2012

  pInfo->twAggSup = (STimeWindowAggSupp){

×

2013

      .waterMark = pIntervalPhyNode->window.watermark,

×

2014

      .calTrigger = pIntervalPhyNode->window.triggerType,

×

2017

      .deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval),

×

2021

  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;

×

2022

  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;

×

2023

  initResultSizeInfo(&pOperator->resultInfo, 4096);

×

2024

  if (pIntervalPhyNode->window.pExprs != NULL) {

×

2025

    int32_t    numOfScalar = 0;

×

2026

    SExprInfo* pScalarExprInfo = NULL;

×

2028

    code = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalar);

×

2029

    QUERY_CHECK_CODE(code, lino, _error);

×

2031

    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);

×

2032

    QUERY_CHECK_CODE(code, lino, _error);

×

2035

  SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);

×

2036

  QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);

×

2037

  initBasicInfo(&pInfo->binfo, pResBlock);

×

2039

  pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState));

×

2040

  QUERY_CHECK_NULL(pInfo->pState, code, lino, _error, terrno);

×

2041

  qInfo("open state %p", pInfo->pState);

×

2042

  pAPI->stateStore.streamStateCopyBackend(pTaskInfo->streamInfo.pState, pInfo->pState);

×

2045

  qInfo("copy state %p to %p", pTaskInfo->streamInfo.pState, pInfo->pState);

×

2047

  pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsIndex);

×

2049

  int32_t    numOfCols = 0;

×

2050

  SExprInfo* pExprInfo = NULL;

×

2051

  code = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);

×

2052

  QUERY_CHECK_CODE(code, lino, _error);

×

2054

  code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,

×

2055

                    pInfo->pState, &pTaskInfo->storageAPI.functionStore);

×

2056

  QUERY_CHECK_CODE(code, lino, _error);

×

2058

  code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

×

2059

  QUERY_CHECK_CODE(code, lino, _error);

×

2060

  initResultRowInfo(&pInfo->binfo.resultRowInfo);

×

2062

  pInfo->numOfChild = numOfChild;

×

2063

  pInfo->pPhyNode = NULL;

×

2064

  code = nodesCloneNode((SNode*)pPhyNode, (SNode**)&pInfo->pPhyNode);

×

2065

  if (TSDB_CODE_SUCCESS != code) {

×

2066

    goto _error;

×

2069

  pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo));

×

2070

  QUERY_CHECK_NULL(pInfo->pPullWins, code, lino, _error, terrno);

×

2071

  pInfo->pullIndex = 0;

×

2072

  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

2073

  pInfo->pPullDataMap = taosHashInit(64, hashFn, true, HASH_NO_LOCK);

×

2074

  pInfo->pFinalPullDataMap = taosHashInit(64, hashFn, true, HASH_NO_LOCK);

×

2076

  code = createSpecialDataBlock(STREAM_RETRIEVE, &pInfo->pPullDataRes);

×

2077

  QUERY_CHECK_CODE(code, lino, _error);

×

2079

  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;

×

2080

  pInfo->ignoreExpiredDataSaved = false;

×

2081

  code = createSpecialDataBlock(STREAM_DELETE_RESULT, &pInfo->pDelRes);

×

2082

  QUERY_CHECK_CODE(code, lino, _error);

×

2084

  pInfo->delIndex = 0;

×

2085

  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));

×

2086

  QUERY_CHECK_NULL(pInfo->pDelWins, code, lino, _error, terrno);

×

2087

  pInfo->delKey.ts = INT64_MAX;

×

2088

  pInfo->delKey.groupId = 0;

×

2089

  pInfo->numOfDatapack = 0;

×

2090

  pInfo->pUpdated = NULL;

×

2091

  pInfo->pUpdatedMap = NULL;

×

2092

  pInfo->stateStore = pTaskInfo->storageAPI.stateStore;

×

2093

  int32_t funResSize = getMaxFunResSize(&pOperator->exprSupp, numOfCols);

×

2094

  pInfo->pState->pFileState = NULL;

×

2097

  pInfo->pState->pResultRowStore.resultRowGet = getResultRowFromBuf;

×

2098

  pInfo->pState->pResultRowStore.resultRowPut = putResultRowToBuf;

×

2099

  pInfo->pState->pExprSupp = &pOperator->exprSupp;

×

2102

      pAPI->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize,

×

2103

                                           compareTs, pInfo->pState, pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo),

×

2104

                                           pHandle->checkpointId, STREAM_STATE_BUFF_HASH, &pInfo->pState->pFileState);

×

2105

  QUERY_CHECK_CODE(code, lino, _error);

×

2107

  pInfo->dataVersion = 0;

×

2108

  pInfo->recvGetAll = false;

×

2109

  pInfo->recvPullover = false;

×

2110

  pInfo->recvRetrive = false;

×

2112

  code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);

×

2113

  QUERY_CHECK_CODE(code, lino, _error);

×

2114

  code = createSpecialDataBlock(STREAM_MID_RETRIEVE, &pInfo->pMidRetriveRes);

×

2115

  QUERY_CHECK_CODE(code, lino, _error);

×

2116

  code = createSpecialDataBlock(STREAM_MID_RETRIEVE, &pInfo->pMidPulloverRes);

×

2117

  QUERY_CHECK_CODE(code, lino, _error);

×

2119

  pInfo->clearState = false;

×

2120

  pInfo->pMidPullDatas = taosArrayInit(4, sizeof(SWinKey));

×

2121

  QUERY_CHECK_NULL(pInfo->pMidPullDatas, code, lino, _error, terrno);

×

2122

  pInfo->pDeletedMap = tSimpleHashInit(4096, hashFn);

×

2123

  QUERY_CHECK_NULL(pInfo->pDeletedMap, code, lino, _error, terrno);

×

2124

  pInfo->destHasPrimaryKey = pIntervalPhyNode->window.destHasPrimaryKey;

×

2125

  pInfo->pOperator = pOperator;

×

2127

  pOperator->operatorType = pPhyNode->type;

×

2128

  if (!IS_FINAL_INTERVAL_OP(pOperator) || numOfChild == 0) {

×

2129

    pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;

×

2131

  pOperator->name = getStreamOpName(pOperator->operatorType);

×

2132

  pOperator->blocking = true;

×

2133

  pOperator->status = OP_NOT_OPENED;

×

2134

  pOperator->info = pInfo;

×

2136

  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {

×

2138

        createOperatorFpSet(NULL, doStreamMidIntervalAggNext, NULL, destroyStreamFinalIntervalOperatorInfo,

×

2142

        createOperatorFpSet(NULL, doStreamFinalIntervalAggNext, NULL, destroyStreamFinalIntervalOperatorInfo,

×

2145

  setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState);

×

2147

  code = initStreamBasicInfo(&pInfo->basic, pOperator);

×

2148

  QUERY_CHECK_CODE(code, lino, _error);

×

2150

  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL ||

×

2151

      pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {

×

2152

    pInfo->basic.primaryPkIndex = -1;

×

2153

    code = initIntervalDownStream(downstream, pPhyNode->type, pInfo, &pInfo->basic);

×

2154

    QUERY_CHECK_CODE(code, lino, _error);

×

2156

  code = appendDownstream(pOperator, &downstream, 1);

×

2157

  QUERY_CHECK_CODE(code, lino, _error);

×

2160

  void*   buff = NULL;

×

2161

  int32_t len = 0;

×

2162

  int32_t res = pAPI->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_CHECKPOINT_NAME,

×

2164

  if (res == TSDB_CODE_SUCCESS) {

×

2165

    doStreamIntervalDecodeOpState(buff, len, pOperator);

×

2166

    taosMemoryFree(buff);

×

2169

  *pOptrInfo = pOperator;

×

2170

  return TSDB_CODE_SUCCESS;

×

2172

_error:

×

2173

  if (pInfo != NULL) destroyStreamFinalIntervalOperatorInfo(pInfo);

×

2174

  destroyOperatorAndDownstreams(pOperator, &downstream, 1);

×

2175

  pTaskInfo->code = code;

×

2176

  return code;

×

2179

void destroyStreamAggSupporter(SStreamAggSupporter* pSup) {

×

2180

  tSimpleHashCleanup(pSup->pResultRows);

×

2181

  destroyDiskbasedBuf(pSup->pResultBuf);

×

2182

  blockDataDestroy(pSup->pScanBlock);

×

2183

  if (pSup->stateStore.streamFileStateDestroy != NULL) {

×

2184

    pSup->stateStore.streamFileStateDestroy(pSup->pState->pFileState);

×

2185

    pSup->pState->pFileState = NULL;

×

2187

  taosMemoryFreeClear(pSup->pState);

×

2188

  taosMemoryFreeClear(pSup->pDummyCtx);

×

2189

×

2191

void destroyResultWinInfo(void* pRes) {

×

2192

  SResultWindowInfo* pWinRes = (SResultWindowInfo*)pRes;

×

2193

  destroyFlusedPos(pWinRes->pStatePos);

×

2194

×

2196

void clearSessionGroupResInfo(SGroupResInfo* pGroupResInfo) {

×

2197

  int32_t size = taosArrayGetSize(pGroupResInfo->pRows);

×

2198

  if (pGroupResInfo->index >= 0 && pGroupResInfo->index < size) {

×

2199

    for (int32_t i = pGroupResInfo->index; i < size; i++) {

×

2200

      SResultWindowInfo* pRes = (SResultWindowInfo*) taosArrayGet(pGroupResInfo->pRows, i);

×

2201

      destroyFlusedPos(pRes->pStatePos);

×

2202

      pRes->pStatePos = NULL;

×

2206

  pGroupResInfo->freeItem = false;

×

2207

  taosArrayDestroy(pGroupResInfo->pRows);

×

2208

  pGroupResInfo->pRows = NULL;

×

2209

  pGroupResInfo->index = 0;

×

2210

×

2212

void destroyStreamSessionAggOperatorInfo(void* param) {

×

2213

  if (param == NULL) {

×

2214

    return;

×

2216

  SStreamSessionAggOperatorInfo* pInfo = (SStreamSessionAggOperatorInfo*)param;

×

2217

  cleanupBasicInfo(&pInfo->binfo);

×

2218

  if (pInfo->pOperator) {

×

2219

    cleanupResultInfoInStream(pInfo->pOperator->pTaskInfo, pInfo->streamAggSup.pState, &pInfo->pOperator->exprSupp,

×

2221

    pInfo->pOperator = NULL;

×

2224

  destroyStreamBasicInfo(&pInfo->basic);

×

2225

  cleanupExprSupp(&pInfo->scalarSupp);

×

2226

  clearSessionGroupResInfo(&pInfo->groupResInfo);

×

2227

  taosArrayDestroyEx(pInfo->pUpdated, destroyResultWinInfo);

×

2228

  pInfo->pUpdated = NULL;

×

2229

  destroyStreamAggSupporter(&pInfo->streamAggSup);

×

2231

  if (pInfo->pChildren != NULL) {

×

2232

    int32_t size = taosArrayGetSize(pInfo->pChildren);

×

2233

    for (int32_t i = 0; i < size; i++) {

×

2234

      SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, i);

×

2235

      destroyOperator(pChild);

×

2237

    taosArrayDestroy(pInfo->pChildren);

×

2240

  colDataDestroy(&pInfo->twAggSup.timeWindowData);

×

2241

  blockDataDestroy(pInfo->pDelRes);

×

2242

  blockDataDestroy(pInfo->pWinBlock);

×

2243

  tSimpleHashCleanup(pInfo->pStUpdated);

×

2244

  tSimpleHashCleanup(pInfo->pStDeleted);

×

2245

  cleanupGroupResInfo(&pInfo->groupResInfo);

×

2247

  taosArrayDestroy(pInfo->historyWins);

×

2248

  blockDataDestroy(pInfo->pCheckpointRes);

×

2249

  tSimpleHashCleanup(pInfo->pPkDeleted);

×

2250

  destroyNonBlockAggSupptor(&pInfo->nbSup);

×

2252

  taosMemoryFreeClear(param);

×

2255

int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfCols,

×

2257

  initBasicInfo(pBasicInfo, pResultBlock);

×

2258

  int32_t code = initExprSupp(pSup, pExprInfo, numOfCols, pStore);

×

2259

  if (code != TSDB_CODE_SUCCESS) {

×

2260

    return code;

×

2263

  for (int32_t i = 0; i < numOfCols; ++i) {

×

2264

    pSup->pCtx[i].saveHandle.pBuf = NULL;

×

2267

  return TSDB_CODE_SUCCESS;

×

2270

void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t nums) {

×

2271

  for (int i = 0; i < nums; i++) {

×

2272

    pDummy[i].functionId = pCtx[i].functionId;

×

2273

    pDummy[i].isNotNullFunc = pCtx[i].isNotNullFunc;

×

2274

    pDummy[i].isPseudoFunc = pCtx[i].isPseudoFunc;

×

2275

    pDummy[i].fpSet.init = pCtx[i].fpSet.init;

×

2277

×

2279

int32_t initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex,

×

2281

  SExecTaskInfo* pTaskInfo = downstream->pTaskInfo;

×

2282

  int32_t        code = TSDB_CODE_SUCCESS;

×

2283

  int32_t        lino = 0;

×

2284

  if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) {

×

2285

    SStreamPartitionOperatorInfo* pScanInfo = downstream->info;

×

2286

    pScanInfo->tsColIndex = tsColIndex;

×

2287

    pBasic->primaryPkIndex = pScanInfo->basic.primaryPkIndex;

×

2290

  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {

×

2291

    code = initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup, pBasic, recalculateInterval);

×

2292

    return code;

×

2294

  SStreamScanInfo* pScanInfo = downstream->info;

×

2295

  pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type};

×

2296

  pScanInfo->pState = pAggSup->pState;

×

2297

  if (!pScanInfo->pUpdateInfo) {

×

2298

    code = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark,

×

2299

                                              pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen,

×

2300

                                              &pScanInfo->pUpdateInfo);

×

2301

    QUERY_CHECK_CODE(code, lino, _end);

×

2303

  pScanInfo->twAggSup = *pTwSup;

×

2304

  pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo;

×

2305

  if (!hasSrcPrimaryKeyCol(pBasic)) {

×

2306

    pBasic->primaryPkIndex = pScanInfo->basic.primaryPkIndex;

×

2309

  pBasic->pTsDataState = pScanInfo->basic.pTsDataState;

×

2311

  if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_SESSION) {

×

2312

    pScanInfo->scanAllTables = true;

×

2314

  pScanInfo->recalculateInterval = recalculateInterval;

×

2315

  pScanInfo->windowSup.parentType = type;

×

2316

  pScanInfo->recParam.gap = pAggSup->gap;

×

2318

_end:

×

2319

  if (code != TSDB_CODE_SUCCESS) {

×

2320

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

2322

  return code;

×

2325

static TSKEY sesionTs(void* pKey) {

×

2326

  SSessionKey* pWinKey = (SSessionKey*)pKey;

×

2327

  return pWinKey->win.skey;

×

2330

int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, int32_t numOfOutput, int64_t gap,

×

2334

  pSup->resultRowSize = (keySize + getResultRowSize(pExpSup->pCtx, numOfOutput)) * ratio;

×

2335

  int32_t lino = 0;

×

2336

  int32_t code = createSpecialDataBlock(STREAM_CLEAR, &pSup->pScanBlock);

×

2337

  QUERY_CHECK_CODE(code, lino, _end);

×

2339

  pSup->gap = gap;

×

2340

  pSup->stateKeySize = keySize;

×

2341

  pSup->stateKeyType = keyType;

×

2342

  pSup->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfOutput, sizeof(SqlFunctionCtx));

×

2343

  QUERY_CHECK_NULL(pSup->pDummyCtx, code, lino, _end, terrno);

×

2345

  pSup->stateStore = *pStore;

×

2346

  pSup->pSessionAPI = pApi;

×

2348

  initDummyFunction(pSup->pDummyCtx, pExpSup->pCtx, numOfOutput);

×

2349

  pSup->pState = taosMemoryCalloc(1, sizeof(SStreamState));

×

2350

  QUERY_CHECK_NULL(pSup->pState, code, lino, _end, terrno);

×

2352

  *(pSup->pState) = *pState;

×

2353

  pSup->stateStore.streamStateSetNumber(pSup->pState, -1, tsIndex);

×

2354

  int32_t funResSize = getMaxFunResSize(pExpSup, numOfOutput);

×

2355

  if (stateType != STREAM_STATE_BUFF_HASH_SORT) {

×

2357

    pSup->pState->pResultRowStore.resultRowGet = getResultRowFromBuf;

×

2358

    pSup->pState->pResultRowStore.resultRowPut = putResultRowToBuf;

×

2359

    pSup->pState->pExprSupp = pExpSup;

×

2362

  if (stateType == STREAM_STATE_BUFF_SORT) {

×

2363

    pSup->pState->pFileState = NULL;

×

2364

    code = pSup->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SSessionKey), pSup->resultRowSize,

×

2365

                                                funResSize, sesionTs, pSup->pState, pTwAggSup->deleteMark, taskIdStr,

×

2366

                                                pHandle->checkpointId, stateType, &pSup->pState->pFileState);

×

2367

  } else if (stateType == STREAM_STATE_BUFF_HASH_SORT || stateType == STREAM_STATE_BUFF_HASH_SEARCH) {

×

2368

    pSup->pState->pFileState = NULL;

×

2369

    code = pSup->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pSup->resultRowSize, funResSize,

×

2370

                                                compareTs, pSup->pState, pTwAggSup->deleteMark, taskIdStr,

×

2371

                                                pHandle->checkpointId, stateType, &pSup->pState->pFileState);

×

2373

  QUERY_CHECK_CODE(code, lino, _end);

×

2375

  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

2376

  pSup->pResultRows = tSimpleHashInit(32, hashFn);

×

2377

  QUERY_CHECK_NULL(pSup->pResultRows, code, lino, _end, terrno);

×

2379

  for (int32_t i = 0; i < numOfOutput; ++i) {

×

2380

    pExpSup->pCtx[i].saveHandle.pState = pSup->pState;

×

2383

  pSup->pCur = NULL;

×

2385

_end:

×

2386

  if (code != TSDB_CODE_SUCCESS) {

×

2387

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

2389

  return code;

×

2392

bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap) {

×

2393

  if (ts + gap >= pWin->skey && ts - gap <= pWin->ekey) {

×

2394

    return true;

×

2396

  return false;

×

2399

bool isInWindow(SResultWindowInfo* pWinInfo, TSKEY ts, int64_t gap) {

×

2400

  return isInTimeWindow(&pWinInfo->sessionWin.win, ts, gap);

×

2403

void getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,

×

2405

  pKey->win.skey = startTs;

×

2406

  pKey->win.ekey = endTs;

×

2407

  pKey->groupId = groupId;

×

2408

  int32_t code = pAggSup->stateStore.streamStateSessionGetKeyByRange(pAggSup->pState, pKey, pKey);

×

2409

  if (code != TSDB_CODE_SUCCESS) {

×

2410

    SET_SESSION_WIN_KEY_INVALID(pKey);

×

2412

×

2414

bool isInvalidSessionWin(SResultWindowInfo* pWinInfo) { return pWinInfo->sessionWin.win.skey == 0; }

×

2416

bool inWinRange(STimeWindow* range, STimeWindow* cur) {

×

2417

  if (cur->skey >= range->skey && cur->ekey <= range->ekey) {

×

2418

    return true;

×

2420

  return false;

×

2423

void clearOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) { pAPI->streamStateClearBuff(pState, pPos); }

×

2425

int32_t setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,

×

2427

  int32_t code = TSDB_CODE_SUCCESS;

×

2428

  int32_t lino = 0;

×

2429

  pCurWin->sessionWin.groupId = groupId;

×

2430

  pCurWin->sessionWin.win.skey = startTs;

×

2431

  pCurWin->sessionWin.win.ekey = endTs;

×

2432

  int32_t size = pAggSup->resultRowSize;

×

2433

  code = pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, pAggSup->gap,

×

2434

                                                             (void**)&pCurWin->pStatePos, &size, pWinCode);

×

2435

  QUERY_CHECK_CODE(code, lino, _end);

×

2437

  if (*pWinCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->sessionWin.win)) {

×

2438

    *pWinCode = TSDB_CODE_FAILED;

×

2439

    clearOutputBuf(pAggSup->pState, pCurWin->pStatePos, &pAggSup->pSessionAPI->stateStore);

×

2442

  if (*pWinCode == TSDB_CODE_SUCCESS) {

×

2443

    pCurWin->isOutput = true;

×

2444

    if (pCurWin->pStatePos->needFree) {

×

2445

      pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->sessionWin);

×

2448

    pCurWin->sessionWin.win.skey = startTs;

×

2449

    pCurWin->sessionWin.win.ekey = endTs;

×

2451

  qDebug("===stream===set session window buff .start:%" PRId64 ",end:%" PRId64 ",groupid:%" PRIu64,

×

2453

_end:

×

2454

  if (code != TSDB_CODE_SUCCESS) {

×

2455

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

2457

  return code;

×

2460

void getSessionWinBuf(SStreamAggSupporter* pAggSup, SStreamStateCur* pCur, SResultWindowInfo* pWinInfo,

×

2462

  int32_t size = 0;

×

2463

  (*pWinCode) = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin,

×

2464

                                                                 (void**)&pWinInfo->pStatePos, &size);

×

2465

  if ((*pWinCode) != TSDB_CODE_SUCCESS) {

×

2466

    return;

×

2469

  pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pCur);

×

2472

int32_t saveDeleteInfo(SArray* pWins, SSessionKey key) {

×

2473

  int32_t code = TSDB_CODE_SUCCESS;

×

2474

  int32_t lino = 0;

×

2475

  void*   res = taosArrayPush(pWins, &key);

×

2476

  if (!res) {

×

2477

    code = terrno;

×

2478

    QUERY_CHECK_CODE(code, lino, _end);

×

2481

_end:

×

2482

  if (code != TSDB_CODE_SUCCESS) {

×

2483

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

2485

  return code;

×

2488

int32_t saveDeleteRes(SSHashObj* pStDelete, SSessionKey key) {

×

2489

  key.win.ekey = key.win.skey;

×

2490

  return tSimpleHashPut(pStDelete, &key, sizeof(SSessionKey), NULL, 0);

×

2493

void releaseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) {

×

2494

  pAPI->streamStateReleaseBuf(pState, pPos, false);

×

2495

×

2497

void removeSessionResult(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey* pKey) {

×

2498

  int32_t     code = TSDB_CODE_SUCCESS;

×

2499

  int32_t     lino = 0;

×

2500

  SSessionKey key = {0};

×

2501

  getSessionHashKey(pKey, &key);

×

2502

  void* pVal = tSimpleHashGet(pHashMap, &key, sizeof(SSessionKey));

×

2503

  if (pVal) {

×

2504

    releaseOutputBuf(pAggSup->pState, *(void**)pVal, &pAggSup->pSessionAPI->stateStore);

×

2505

    int32_t tmpRes = tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey));

×

2506

    qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

2508

  int32_t tmpRes = tSimpleHashRemove(pResMap, &key, sizeof(SSessionKey));

×

2509

  qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

2510

×

2512

void getSessionHashKey(const SSessionKey* pKey, SSessionKey* pHashKey) {

×

2513

  *pHashKey = *pKey;

×

2514

  pHashKey->win.ekey = pKey->win.skey;

×

2515

×

2517

void removeSessionDeleteResults(SSHashObj* pHashMap, SArray* pWins) {

×

2518

  if (tSimpleHashGetSize(pHashMap) == 0) {

×

2519

    return;

×

2521

  int32_t size = taosArrayGetSize(pWins);

×

2522

  for (int32_t i = 0; i < size; i++) {

×

2523

    SResultWindowInfo* pWin = taosArrayGet(pWins, i);

×

2524

    if (!pWin) continue;

×

2525

    SSessionKey key = {0};

×

2526

    getSessionHashKey(&pWin->sessionWin, &key);

×

2527

    int32_t tmpRes = tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey));

×

2528

    qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

2532

void removeSessionResults(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SArray* pWins) {

×

2533

  if (tSimpleHashGetSize(pHashMap) == 0) {

×

2534

    return;

×

2536

  int32_t size = taosArrayGetSize(pWins);

×

2537

  for (int32_t i = 0; i < size; i++) {

×

2538

    SSessionKey* pWin = taosArrayGet(pWins, i);

×

2539

    if (!pWin) continue;

×

2540

    SSessionKey key = {0};

×

2541

    getSessionHashKey(pWin, &key);

×

2542

    void* pVal = tSimpleHashGet(pHashMap, &key, sizeof(SSessionKey));

×

2543

    if (pVal) {

×

2544

      releaseOutputBuf(pAggSup->pState, *(void**)pVal, &pAggSup->pSessionAPI->stateStore);

×

2545

      int32_t tmpRes = tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey));

×

2546

      qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

2551

int32_t updateSessionWindowInfo(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo, TSKEY* pStartTs,

×

2555

  int32_t code = TSDB_CODE_SUCCESS;

×

2556

  int32_t lino = 0;

×

2557

  for (int32_t i = start; i < rows; ++i) {

×

2558

    if (!isInWindow(pWinInfo, pStartTs[i], gap) && (!pEndTs || !isInWindow(pWinInfo, pEndTs[i], gap))) {

×

2559

      (*pWinRos) = i - start;

×

2560

      goto _end;

×

2562

    if (pWinInfo->sessionWin.win.skey > pStartTs[i]) {

×

2563

      if (pStDeleted && pWinInfo->isOutput) {

×

2564

        code = saveDeleteRes(pStDeleted, pWinInfo->sessionWin);

×

2565

        QUERY_CHECK_CODE(code, lino, _end);

×

2568

      removeSessionResult(pAggSup, pStUpdated, pResultRows, &pWinInfo->sessionWin);

×

2569

      pWinInfo->sessionWin.win.skey = pStartTs[i];

×

2571

    pWinInfo->sessionWin.win.ekey = TMAX(pWinInfo->sessionWin.win.ekey, pStartTs[i]);

×

2572

    if (pEndTs) {

×

2573

      pWinInfo->sessionWin.win.ekey = TMAX(pWinInfo->sessionWin.win.ekey, pEndTs[i]);

×

2575

    memcpy(pWinInfo->pStatePos->pKey, &pWinInfo->sessionWin, sizeof(SSessionKey));

×

2577

  (*pWinRos) = rows - start;

×

2579

_end:

×

2580

  if (code != TSDB_CODE_SUCCESS) {

×

2581

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

2583

  return code;

×

2586

static int32_t initSessionOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, SqlFunctionCtx* pCtx,

×

2588

  *pResult = (SResultRow*)pWinInfo->pStatePos->pRowBuff;

×

2590

  (*pResult)->win = pWinInfo->sessionWin.win;

×

2591

  return setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowEntryInfoOffset);

×

2594

int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindowInfo* pCurWin, SResultRow** pResult,

×

2597

  int32_t        code = TSDB_CODE_SUCCESS;

×

2598

  int32_t        lino = 0;

×

2599

  SExprSupp*     pSup = &pOperator->exprSupp;

×

2600

  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

×

2601

  code = initSessionOutputBuf(pCurWin, pResult, pSup->pCtx, numOutput, pSup->rowEntryInfoOffset);

×

2602

  QUERY_CHECK_CODE(code, lino, _end);

×

2604

  updateTimeWindowInfo(pTimeWindowData, &pCurWin->sessionWin.win, winDelta);

×

2605

  code = applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, pTimeWindowData, startIndex, winRows, rows, numOutput);

×

2607

_end:

×

2608

  if (code != TSDB_CODE_SUCCESS) {

×

2609

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

2611

  return code;

×

2614

void doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey) {

×

2615

  pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, pKey);

×

2616

  SSessionKey hashKey = {0};

×

2617

  getSessionHashKey(pKey, &hashKey);

×

2618

  int32_t tmpRes = tSimpleHashRemove(pAggSup->pResultRows, &hashKey, sizeof(SSessionKey));

×

2619

  qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

2620

×

2622

void setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo* pWinInfo) {

×

2623

  void* pVal = tSimpleHashGet(pStUpdated, &pWinInfo->sessionWin, sizeof(SSessionKey));

×

2624

  if (pVal) {

×

2625

    SResultWindowInfo* pWin = pVal;

×

2626

    pWinInfo->isOutput = pWin->isOutput;

×

2628

×

2630

void getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin,

×

2632

  SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->sessionWin);

×

2633

  pNextWin->isOutput = true;

×

2634

  setSessionWinOutputInfo(pStUpdated, pNextWin);

×

2635

  int32_t size = 0;

×

2636

  pNextWin->sessionWin = pCurWin->sessionWin;

×

2637

  int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->sessionWin,

×

2638

                                                                  (void**)&pNextWin->pStatePos, &size);

×

2639

  if (code != TSDB_CODE_SUCCESS) {

×

2640

    SET_SESSION_WIN_INVALID(*pNextWin);

×

2642

  pAggSup->stateStore.streamStateFreeCur(pCur);

×

2643

×

2645

int32_t compactTimeWindow(SExprSupp* pSup, SStreamAggSupporter* pAggSup, STimeWindowAggSupp* pTwAggSup,

×

2648

  int32_t     code = TSDB_CODE_SUCCESS;

×

2649

  int32_t     lino = 0;

×

2650

  SResultRow* pCurResult = NULL;

×

2651

  int32_t     numOfOutput = pSup->numOfExprs;

×

2652

  code = initSessionOutputBuf(pCurWin, &pCurResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset);

×

2653

  QUERY_CHECK_CODE(code, lino, _end);

×

2655

  SResultRow* pWinResult = NULL;

×

2656

  code = initSessionOutputBuf(pNextWin, &pWinResult, pAggSup->pDummyCtx, numOfOutput, pSup->rowEntryInfoOffset);

×

2657

  QUERY_CHECK_CODE(code, lino, _end);

×

2659

  pCurWin->sessionWin.win.ekey = TMAX(pCurWin->sessionWin.win.ekey, pNextWin->sessionWin.win.ekey);

×

2660

  memcpy(pCurWin->pStatePos->pKey, &pCurWin->sessionWin, sizeof(SSessionKey));

×

2662

  int64_t winDelta = 0;

×

2663

  if (addGap) {

×

2664

    winDelta = pAggSup->gap;

×

2666

  updateTimeWindowInfo(&pTwAggSup->timeWindowData, &pCurWin->sessionWin.win, winDelta);

×

2667

  code = compactFunctions(pSup->pCtx, pAggSup->pDummyCtx, numOfOutput, pTaskInfo, &pTwAggSup->timeWindowData);

×

2668

  QUERY_CHECK_CODE(code, lino, _end);

×

2670

  int32_t tmpRes = tSimpleHashRemove(pStUpdated, &pNextWin->sessionWin, sizeof(SSessionKey));

×

2671

  qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

2673

  if (pNextWin->isOutput && pStDeleted) {

×

2674

    qDebug("===stream=== save delete window info %" PRId64 ", %" PRIu64, pNextWin->sessionWin.win.skey,

×

2676

    code = saveDeleteRes(pStDeleted, pNextWin->sessionWin);

×

2677

    QUERY_CHECK_CODE(code, lino, _end);

×

2679

  removeSessionResult(pAggSup, pStUpdated, pAggSup->pResultRows, &pNextWin->sessionWin);

×

2680

  doDeleteSessionWindow(pAggSup, &pNextWin->sessionWin);

×

2681

  releaseOutputBuf(pAggSup->pState, pNextWin->pStatePos, &pAggSup->pSessionAPI->stateStore);

×

2683

_end:

×

2684

  if (code != TSDB_CODE_SUCCESS) {

×

2685

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

2687

  return code;

×

2690

int32_t compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SSHashObj* pStUpdated,

×

2692

  int32_t                        code = TSDB_CODE_SUCCESS;

×

2693

  int32_t                        lino = 0;

×

2694

  SExprSupp*                     pSup = &pOperator->exprSupp;

×

2695

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

2696

  SStorageAPI*                   pAPI = &pOperator->pTaskInfo->storageAPI;

×

2697

  int32_t                        winNum = 0;

×

2698

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

2699

  SResultRow*                    pCurResult = NULL;

×

2700

  int32_t                        numOfOutput = pOperator->exprSupp.numOfExprs;

×

2701

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

2703

  if (pIsEnd != NULL) {

×

2704

    (*pIsEnd) = false;

×

2707

  while (1) {

×

2708

    SResultWindowInfo winInfo = {0};

×

2709

    getNextSessionWinInfo(pAggSup, pStUpdated, pCurWin, &winInfo);

×

2710

    if (!IS_VALID_SESSION_WIN(winInfo)) {

×

2711

      if (pIsEnd != NULL) {

×

2712

        (*pIsEnd) = true;

×

2714

      releaseOutputBuf(pAggSup->pState, winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);

×

2715

      break;

×

2717

    if (!isInWindow(pCurWin, winInfo.sessionWin.win.skey, pAggSup->gap) ||

×

2718

        !inWinRange(&pAggSup->winRange, &winInfo.sessionWin.win)) {

×

2719

      releaseOutputBuf(pAggSup->pState, winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);

×

2720

      break;

×

2723

        compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, pCurWin, &winInfo, pStUpdated, pStDeleted, true);

×

2724

    QUERY_CHECK_CODE(code, lino, _end);

×

2725

    winNum++;

×

2727

  if (pWinNum) {

×

2728

    (*pWinNum) = winNum;

×

2731

_end:

×

2732

  if (code != TSDB_CODE_SUCCESS) {

×

2733

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

2735

  return code;

×

2738

static void compactSessionSemiWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin) {

×

2739

  SExprSupp*                     pSup = &pOperator->exprSupp;

×

2740

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

2741

  SStorageAPI*                   pAPI = &pOperator->pTaskInfo->storageAPI;

×

2742

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

2743

  SResultRow*                    pCurResult = NULL;

×

2744

  int32_t                        numOfOutput = pOperator->exprSupp.numOfExprs;

×

2745

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

2747

  while (1) {

×

2748

    SResultWindowInfo winInfo = {0};

×

2749

    getNextSessionWinInfo(pAggSup, NULL, pCurWin, &winInfo);

×

2750

    if (!IS_VALID_SESSION_WIN(winInfo) || !isInWindow(pCurWin, winInfo.sessionWin.win.skey, pAggSup->gap) ||

×

2751

        !inWinRange(&pAggSup->winRange, &winInfo.sessionWin.win)) {

×

2752

      releaseOutputBuf(pAggSup->pState, winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);

×

2753

      break;

×

2755

    pCurWin->sessionWin.win.ekey = TMAX(pCurWin->sessionWin.win.ekey, winInfo.sessionWin.win.ekey);

×

2756

    memcpy(pCurWin->pStatePos->pKey, &pCurWin->sessionWin, sizeof(SSessionKey));

×

2757

    doDeleteSessionWindow(pAggSup, &winInfo.sessionWin);

×

2758

    releaseOutputBuf(pAggSup->pState, winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);

×

2760

×

2762

int32_t saveSessionOutputBuf(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo) {

×

2763

  qDebug("===stream===try save session result skey:%" PRId64 ", ekey:%" PRId64 ".pos%d", pWinInfo->sessionWin.win.skey,

×

2765

  return pAggSup->stateStore.streamStateSessionPut(pAggSup->pState, &pWinInfo->sessionWin, pWinInfo->pStatePos,

×

2769

static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pStUpdated,

×

2771

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

2772

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

2773

  int32_t                        numOfOutput = pOperator->exprSupp.numOfExprs;

×

2774

  uint64_t                       groupId = pSDataBlock->info.id.groupId;

×

2775

  int32_t                        code = TSDB_CODE_SUCCESS;

×

2776

  int32_t                        lino = 0;

×

2777

  SResultRow*                    pResult = NULL;

×

2778

  int32_t                        rows = pSDataBlock->info.rows;

×

2779

  int32_t                        winRows = 0;

×

2780

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

2782

  pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);

×

2783

  pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;

×

2784

  if (pAggSup->winRange.ekey <= 0) {

×

2785

    pAggSup->winRange.ekey = INT64_MAX;

×

2788

  SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);

×

2789

  TSKEY*           startTsCols = (int64_t*)pStartTsCol->pData;

×

2790

  SColumnInfoData* pEndTsCol = NULL;

×

2791

  if (hasEndTs) {

×

2792

    pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->endTsIndex);

×

2794

    pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);

×

2797

  TSKEY* endTsCols = (int64_t*)pEndTsCol->pData;

×

2799

  void*            pPkVal = NULL;

×

2800

  int32_t          pkLen = 0;

×

2801

  SColumnInfoData* pPkColDataInfo = NULL;

×

2802

  if (hasSrcPrimaryKeyCol(&pInfo->basic)) {

×

2803

    pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex);

×

2806

  for (int32_t i = 0; i < rows;) {

×

2807

    if (hasSrcPrimaryKeyCol(&pInfo->basic) && !IS_FINAL_SESSION_OP(pOperator) && pInfo->ignoreExpiredData) {

×

2808

      pPkVal = colDataGetData(pPkColDataInfo, i);

×

2809

      pkLen = colDataGetRowLength(pPkColDataInfo, i);

×

2811

    if (!IS_FINAL_SESSION_OP(pOperator) && pInfo->ignoreExpiredData &&

×

2812

        checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo, &pInfo->twAggSup,

×

2813

                         pSDataBlock->info.id.uid, endTsCols[i], pPkVal, pkLen)) {

×

2814

      i++;

×

2815

      continue;

×

2817

    SResultWindowInfo winInfo = {0};

×

2818

    int32_t           winCode = TSDB_CODE_SUCCESS;

×

2819

    code = setSessionOutputBuf(pAggSup, startTsCols[i], endTsCols[i], groupId, &winInfo, &winCode);

×

2820

    QUERY_CHECK_CODE(code, lino, _end);

×

2822

    if (winCode != TSDB_CODE_SUCCESS && IS_NORMAL_SESSION_OP(pOperator) &&

×

2823

        BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {

×

2824

      code = addSessionAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &winInfo.sessionWin, &pInfo->basic.notifyEventSup,

×

2826

      QUERY_CHECK_CODE(code, lino, _end);

×

2830

    if (!winInfo.pStatePos) {

×

2831

      continue;

×

2833

    setSessionWinOutputInfo(pStUpdated, &winInfo);

×

2834

    code = updateSessionWindowInfo(pAggSup, &winInfo, startTsCols, endTsCols, groupId, rows, i, pAggSup->gap,

×

2836

    QUERY_CHECK_CODE(code, lino, _end);

×

2838

    int64_t winDelta = 0;

×

2839

    if (addGap) {

×

2840

      winDelta = pAggSup->gap;

×

2842

    code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &winInfo, &pResult, i, winRows, rows, numOfOutput,

×

2844

    QUERY_CHECK_CODE(code, lino, _end);

×

2846

    code = compactSessionWindow(pOperator, &winInfo, pStUpdated, pStDeleted, addGap, NULL, NULL);

×

2847

    QUERY_CHECK_CODE(code, lino, _end);

×

2849

    code = saveSessionOutputBuf(pAggSup, &winInfo);

×

2850

    QUERY_CHECK_CODE(code, lino, _end);

×

2852

    if (pInfo->destHasPrimaryKey && winInfo.isOutput && IS_NORMAL_SESSION_OP(pOperator)) {

×

2853

      code = saveDeleteRes(pInfo->pPkDeleted, winInfo.sessionWin);

×

2854

      QUERY_CHECK_CODE(code, lino, _end);

×

2856

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) {

×

2857

      code = saveResult(winInfo, pStUpdated);

×

2858

      QUERY_CHECK_CODE(code, lino, _end);

×

2860

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

2861

      winInfo.pStatePos->beUpdated = true;

×

2862

      SSessionKey key = {0};

×

2863

      getSessionHashKey(&winInfo.sessionWin, &key);

×

2864

      code = tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo));

×

2865

      QUERY_CHECK_CODE(code, lino, _end);

×

2868

    i += winRows;

×

2871

_end:

×

2872

  if (code != TSDB_CODE_SUCCESS) {

×

2873

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

2875

×

2877

int32_t doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result) {

×

2878

  int32_t          code = TSDB_CODE_SUCCESS;

×

2879

  int32_t          lino = 0;

×

2880

  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);

×

2881

  TSKEY*           startDatas = (TSKEY*)pStartTsCol->pData;

×

2882

  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);

×

2883

  TSKEY*           endDatas = (TSKEY*)pEndTsCol->pData;

×

2884

  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);

×

2885

  uint64_t*        gpDatas = (uint64_t*)pGroupCol->pData;

×

2886

  for (int32_t i = 0; i < pBlock->info.rows; i++) {

×

2887

    while (1) {

×

2888

      SSessionKey curWin = {0};

×

2889

      getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], &curWin);

×

2890

      if (IS_INVALID_SESSION_WIN_KEY(curWin)) {

×

2891

        break;

×

2893

      doDeleteSessionWindow(pAggSup, &curWin);

×

2894

      if (result) {

×

2895

        code = saveDeleteInfo(result, curWin);

×

2896

        QUERY_CHECK_CODE(code, lino, _end);

×

2901

_end:

×

2902

  if (code != TSDB_CODE_SUCCESS) {

×

2903

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

2905

  return code;

×

2908

inline int32_t sessionKeyCompareAsc(const void* pKey1, const void* pKey2) {

×

2909

  SResultWindowInfo* pWinInfo1 = (SResultWindowInfo*)pKey1;

×

2910

  SResultWindowInfo* pWinInfo2 = (SResultWindowInfo*)pKey2;

×

2911

  SSessionKey*       pWin1 = &pWinInfo1->sessionWin;

×

2912

  SSessionKey*       pWin2 = &pWinInfo2->sessionWin;

×

2914

  if (pWin1->groupId > pWin2->groupId) {

×

2915

    return 1;

×

2916

  } else if (pWin1->groupId < pWin2->groupId) {

×

2917

    return -1;

×

2920

  if (pWin1->win.skey > pWin2->win.skey) {

×

2921

    return 1;

×

2922

  } else if (pWin1->win.skey < pWin2->win.skey) {

×

2923

    return -1;

×

2926

  return 0;

×

2929

static int32_t appendToDeleteDataBlock(SOperatorInfo* pOp, SSDataBlock *pBlock, SSessionKey *pKey) {

×

2930

  int32_t        code = TSDB_CODE_SUCCESS;

×

2931

  int32_t        lino = 0;

×

2932

  SExecTaskInfo* pTaskInfo = pOp->pTaskInfo;

×

2934

  QUERY_CHECK_NULL(pBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);

×

2935

  QUERY_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA);

×

2937

  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);

×

2938

  code = colDataSetVal(pStartTsCol, pBlock->info.rows, (const char*)&pKey->win.skey, false);

×

2939

  QUERY_CHECK_CODE(code, lino, _end);

×

2941

  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);

×

2942

  code = colDataSetVal(pEndTsCol, pBlock->info.rows, (const char*)&pKey->win.skey, false);

×

2943

  QUERY_CHECK_CODE(code, lino, _end);

×

2945

  SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);

×

2946

  colDataSetNULL(pUidCol, pBlock->info.rows);

×

2948

  SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);

×

2949

  code = colDataSetVal(pGpCol, pBlock->info.rows, (const char*)&pKey->groupId, false);

×

2950

  QUERY_CHECK_CODE(code, lino, _end);

×

2952

  SColumnInfoData* pCalStCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);

×

2953

  colDataSetNULL(pCalStCol, pBlock->info.rows);

×

2955

  SColumnInfoData* pCalEdCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);

×

2956

  colDataSetNULL(pCalEdCol, pBlock->info.rows);

×

2958

  SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);

×

2959

  if (!pTableCol) {

×

2960

    QUERY_CHECK_CODE(code, lino, _end);

×

2963

  void*        tbname = NULL;

×

2964

  int32_t      winCode = TSDB_CODE_SUCCESS;

×

2965

  SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI;

×

2967

      pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, pKey->groupId, &tbname, false, &winCode);

×

2968

  QUERY_CHECK_CODE(code, lino, _end);

×

2970

  if (winCode != TSDB_CODE_SUCCESS) {

×

2971

    colDataSetNULL(pTableCol, pBlock->info.rows);

×

2974

    STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));

×

2975

    code = colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false);

×

2976

    QUERY_CHECK_CODE(code, lino, _end);

×

2977

    pAPI->stateStore.streamStateFreeVal(tbname);

×

2979

  pBlock->info.rows += 1;

×

2981

_end:

×

2982

  if (code != TSDB_CODE_SUCCESS) {

×

2983

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

2985

  return code;

×

2988

void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite,

×

2990

  int32_t        code = TSDB_CODE_SUCCESS;

×

2991

  int32_t        lino = 0;

×

2992

  SExecTaskInfo* pTaskInfo = pOp->pTaskInfo;

×

2993

  int64_t        minWindowSize = getMinWindowSize(pOp);

×

2994

  int32_t        numOfRows = getNumOfTotalRes(pGroupResInfo);

×

2996

  blockDataCleanup(pBlock);

×

2997

  int32_t size = tSimpleHashGetSize(pStDeleted);

×

2998

  if (minWindowSize > 0) {

×

3000

    for (int32_t i = pGroupResInfo->delIndex; i < numOfRows; ++i) {

×

3001

      SResultWindowInfo* pWinInfo = taosArrayGet(pGroupResInfo->pRows, i);

×

3002

      SRowBuffPos*       pPos = pWinInfo->pStatePos;

×

3003

      SSessionKey*       pKey = (SSessionKey*)pPos->pKey;

×

3004

      if (pKey->win.ekey - pKey->win.skey < minWindowSize) {

×

3005

        size++;

×

3009

  if (size == 0) {

×

3010

    return;

×

3012

  code = blockDataEnsureCapacity(pBlock, size);

×

3013

  QUERY_CHECK_CODE(code, lino, _end);

×

3015

  int32_t iter = 0;

×

3016

  while (((*Ite) = tSimpleHashIterate(pStDeleted, *Ite, &iter)) != NULL) {

×

3017

    if (pBlock->info.rows + 1 > pBlock->info.capacity) {

×

3018

      break;

×

3020

    SSessionKey*     res = tSimpleHashGetKey(*Ite, NULL);

×

3021

    code = appendToDeleteDataBlock(pOp, pBlock, res);

×

3022

    QUERY_CHECK_CODE(code, lino, _end);

×

3025

  if (minWindowSize > 0) {

×

3026

    for (int32_t i = pGroupResInfo->delIndex; i < numOfRows; ++i) {

×

3027

      SResultWindowInfo* pWinInfo = taosArrayGet(pGroupResInfo->pRows, i);

×

3028

      SRowBuffPos*       pPos = pWinInfo->pStatePos;

×

3029

      SSessionKey*       pKey = (SSessionKey*)pPos->pKey;

×

3030

      if (pKey->win.ekey - pKey->win.skey < minWindowSize) {

×

3031

        code = appendToDeleteDataBlock(pOp, pBlock, pKey);

×

3032

        QUERY_CHECK_CODE(code, lino, _end);

×

3035

    pGroupResInfo->delIndex = numOfRows;

×

3038

_end:

×

3039

  if ((*Ite) == NULL) {

×

3040

    tSimpleHashClear(pStDeleted);

×

3043

  if (code != TSDB_CODE_SUCCESS) {

×

3044

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3048

static int32_t rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SSHashObj* pStUpdated) {

×

3049

  int32_t        winCode = TSDB_CODE_SUCCESS;

×

3050

  int32_t        code = TSDB_CODE_SUCCESS;

×

3051

  int32_t        lino = 0;

×

3052

  SExprSupp*     pSup = &pOperator->exprSupp;

×

3053

  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

×

3054

  SStorageAPI*   pAPI = &pOperator->pTaskInfo->storageAPI;

×

3056

  int32_t                        size = taosArrayGetSize(pWinArray);

×

3057

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3058

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

3059

  int32_t                        numOfOutput = pSup->numOfExprs;

×

3060

  int32_t                        numOfChild = taosArrayGetSize(pInfo->pChildren);

×

3062

  for (int32_t i = 0; i < size; i++) {

×

3063

    SSessionKey*      pWinKey = taosArrayGet(pWinArray, i);

×

3064

    int32_t           num = 0;

×

3065

    SResultWindowInfo parentWin = {0};

×

3066

    for (int32_t j = 0; j < numOfChild; j++) {

×

3067

      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, j);

×

3068

      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;

×

3069

      SStreamAggSupporter*           pChAggSup = &pChInfo->streamAggSup;

×

3070

      SSessionKey                    chWinKey = {0};

×

3071

      getSessionHashKey(pWinKey, &chWinKey);

×

3072

      SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pChAggSup->pState, &chWinKey);

×

3073

      SResultRow*      pResult = NULL;

×

3074

      SResultRow*      pChResult = NULL;

×

3075

      while (1) {

×

3076

        SResultWindowInfo childWin = {0};

×

3077

        childWin.sessionWin = *pWinKey;

×

3078

        getSessionWinBuf(pChAggSup, pCur, &childWin, &winCode);

×

3080

        if (winCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &childWin.sessionWin.win)) {

×

3081

          releaseOutputBuf(pAggSup->pState, childWin.pStatePos, &pAggSup->stateStore);

×

3082

          continue;

×

3085

        if (winCode == TSDB_CODE_SUCCESS && inWinRange(&pWinKey->win, &childWin.sessionWin.win)) {

×

3086

          if (num == 0) {

×

3087

            code = setSessionOutputBuf(pAggSup, pWinKey->win.skey, pWinKey->win.ekey, pWinKey->groupId, &parentWin,

×

3089

            QUERY_CHECK_CODE(code, lino, _end);

×

3091

            parentWin.sessionWin = childWin.sessionWin;

×

3092

            memcpy(parentWin.pStatePos->pKey, &parentWin.sessionWin, sizeof(SSessionKey));

×

3093

            code = initSessionOutputBuf(&parentWin, &pResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset);

×

3094

            QUERY_CHECK_CODE(code, lino, _end);

×

3096

          num++;

×

3097

          parentWin.sessionWin.win.skey = TMIN(parentWin.sessionWin.win.skey, childWin.sessionWin.win.skey);

×

3098

          parentWin.sessionWin.win.ekey = TMAX(parentWin.sessionWin.win.ekey, childWin.sessionWin.win.ekey);

×

3099

          memcpy(parentWin.pStatePos->pKey, &parentWin.sessionWin, sizeof(SSessionKey));

×

3101

          updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin.sessionWin.win, pAggSup->gap);

×

3102

          code = initSessionOutputBuf(&childWin, &pChResult, pChild->exprSupp.pCtx, numOfOutput,

×

3104

          QUERY_CHECK_CODE(code, lino, _end);

×

3106

          code = compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo,

×

3108

          QUERY_CHECK_CODE(code, lino, _end);

×

3110

          code = compactSessionWindow(pOperator, &parentWin, pStUpdated, NULL, true, NULL, NULL);

×

3111

          QUERY_CHECK_CODE(code, lino, _end);

×

3113

          releaseOutputBuf(pAggSup->pState, childWin.pStatePos, &pAggSup->stateStore);

×

3115

          releaseOutputBuf(pAggSup->pState, childWin.pStatePos, &pAggSup->stateStore);

×

3116

          break;

×

3119

      pAPI->stateStore.streamStateFreeCur(pCur);

×

3121

    if (num > 0) {

×

3122

      code = saveResult(parentWin, pStUpdated);

×

3123

      QUERY_CHECK_CODE(code, lino, _end);

×

3125

      code = saveSessionOutputBuf(pAggSup, &parentWin);

×

3126

      QUERY_CHECK_CODE(code, lino, _end);

×

3130

_end:

×

3131

  if (code != TSDB_CODE_SUCCESS) {

×

3132

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3134

  return code;

×

3137

int32_t closeSessionWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SSHashObj* pClosed) {

×

3138

  int32_t code = TSDB_CODE_SUCCESS;

×

3139

  int32_t lino = 0;

×

3140

  void*   pIte = NULL;

×

3141

  int32_t iter = 0;

×

3142

  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {

×

3143

    SResultWindowInfo* pWinInfo = pIte;

×

3144

    if (isCloseWindow(&pWinInfo->sessionWin.win, pTwSup)) {

×

3145

      if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE && pClosed) {

×

3146

        code = saveResult(*pWinInfo, pClosed);

×

3147

        QUERY_CHECK_CODE(code, lino, _end);

×

3149

      SSessionKey* pKey = tSimpleHashGetKey(pIte, NULL);

×

3150

      code = tSimpleHashIterateRemove(pHashMap, pKey, sizeof(SSessionKey), &pIte, &iter);

×

3151

      QUERY_CHECK_CODE(code, lino, _end);

×

3154

_end:

×

3155

  if (code != TSDB_CODE_SUCCESS) {

×

3156

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3158

  return code;

×

3161

static int32_t closeChildSessionWindow(SArray* pChildren, TSKEY maxTs) {

×

3162

  int32_t code = TSDB_CODE_SUCCESS;

×

3163

  int32_t lino = 0;

×

3165

  int32_t size = taosArrayGetSize(pChildren);

×

3166

  for (int32_t i = 0; i < size; i++) {

×

3167

    SOperatorInfo*                 pChildOp = taosArrayGetP(pChildren, i);

×

3168

    SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info;

×

3169

    pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);

×

3170

    code = closeSessionWindow(pChInfo->streamAggSup.pResultRows, &pChInfo->twAggSup, NULL);

×

3171

    QUERY_CHECK_CODE(code, lino, _end);

×

3173

_end:

×

3174

  if (code != TSDB_CODE_SUCCESS) {

×

3175

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3177

  return code;

×

3180

int32_t getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated) {

×

3181

  int32_t code = TSDB_CODE_SUCCESS;

×

3182

  int32_t lino = 0;

×

3183

  void*   pIte = NULL;

×

3184

  int32_t iter = 0;

×

3185

  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {

×

3186

    SResultWindowInfo* pWinInfo = pIte;

×

3187

    if (!pWinInfo->pStatePos->beUpdated) {

×

3188

      continue;

×

3190

    pWinInfo->pStatePos->beUpdated = false;

×

3191

    code = saveResult(*pWinInfo, pStUpdated);

×

3192

    QUERY_CHECK_CODE(code, lino, _end);

×

3195

_end:

×

3196

  if (code != TSDB_CODE_SUCCESS) {

×

3197

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3199

  return code;

×

3202

int32_t copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted) {

×

3203

  int32_t code = TSDB_CODE_SUCCESS;

×

3204

  int32_t lino = 0;

×

3205

  int32_t size = taosArrayGetSize(pResWins);

×

3206

  for (int32_t i = 0; i < size; i++) {

×

3207

    SSessionKey* pWinKey = taosArrayGet(pResWins, i);

×

3208

    if (!pWinKey) continue;

×

3209

    SSessionKey winInfo = {0};

×

3210

    getSessionHashKey(pWinKey, &winInfo);

×

3211

    code = tSimpleHashPut(pStDeleted, &winInfo, sizeof(SSessionKey), NULL, 0);

×

3212

    QUERY_CHECK_CODE(code, lino, _end);

×

3215

_end:

×

3216

  if (code != TSDB_CODE_SUCCESS) {

×

3217

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3219

  return code;

×

3223

void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList) {

×

3224

  pGroupResInfo->pRows = pArrayList;

×

3225

  pGroupResInfo->index = 0;

×

3226

  pGroupResInfo->pBuf = NULL;

×

3227

  pGroupResInfo->freeItem = false;

×

3228

  pGroupResInfo->delIndex = 0;

×

3229

×

3231

int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup,

×

3233

  int32_t         code = TSDB_CODE_SUCCESS;

×

3234

  int32_t         lino = 0;

×

3235

  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;

×

3236

  SStorageAPI*    pAPI = &pTaskInfo->storageAPI;

×

3237

  SExprInfo*      pExprInfo = pSup->pExprInfo;

×

3238

  int32_t         numOfExprs = pSup->numOfExprs;

×

3239

  int32_t*        rowEntryOffset = pSup->rowEntryInfoOffset;

×

3240

  SqlFunctionCtx* pCtx = pSup->pCtx;

×

3241

  int64_t         minWindowSize = getMinWindowSize(pOperator);

×

3243

  int32_t numOfRows = getNumOfTotalRes(pGroupResInfo);

×

3245

  for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) {

×

3246

    SResultWindowInfo* pWinInfo = taosArrayGet(pGroupResInfo->pRows, i);

×

3247

    SRowBuffPos*       pPos = pWinInfo->pStatePos;

×

3248

    SResultRow*        pRow = NULL;

×

3249

    SSessionKey*       pKey = (SSessionKey*)pPos->pKey;

×

3251

    if (pBlock->info.id.groupId == 0) {

×

3252

      pBlock->info.id.groupId = pKey->groupId;

×

3254

      void*   tbname = NULL;

×

3255

      int32_t winCode = TSDB_CODE_SUCCESS;

×

3256

      code = pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId,

×

3258

      QUERY_CHECK_CODE(code, lino, _end);

×

3260

      if (winCode != TSDB_CODE_SUCCESS) {

×

3261

        pBlock->info.parTbName[0] = 0;

×

3263

        memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);

×

3266

      qDebug("%s partName:%s, groupId:%"PRIu64, __FUNCTION__, (char*)tbname, pKey->groupId);

×

3267

      pAPI->stateStore.streamStateFreeVal(tbname);

×

3270

      if (pBlock->info.id.groupId != pKey->groupId) {

×

3271

        break;

×

3275

    code = pAPI->stateStore.streamStateGetByPos(pState, pPos, (void**)&pRow);

×

3276

    QUERY_CHECK_CODE(code, lino, _end);

×

3278

    doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset);

×

3280

    if (pRow->numOfRows == 0) {

×

3281

      pGroupResInfo->index += 1;

×

3282

      continue;

×

3285

    if (pKey->win.ekey - pKey->win.skey < minWindowSize) {

×

3286

      qDebug("skip small window, groupId: %" PRId64 ", windowSize: %" PRId64 ", minWindowSize: %" PRId64, pKey->groupId,

×

3288

      pGroupResInfo->index += 1;

×

3289

      continue;

×

3292

    if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) {

×

3293

      break;

×

3296

    pGroupResInfo->index += 1;

×

3298

    for (int32_t j = 0; j < numOfExprs; ++j) {

×

3299

      int32_t slotId = pExprInfo[j].base.resSchema.slotId;

×

3301

      pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset);

×

3302

      if (pCtx[j].fpSet.finalize) {

×

3303

        int32_t tmpRes = pCtx[j].fpSet.finalize(&pCtx[j], pBlock);

×

3304

        if (TAOS_FAILED(tmpRes)) {

×

3305

          qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(tmpRes));

×

3306

          QUERY_CHECK_CODE(code, lino, _end);

×

3308

      } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) {

×

3313

        SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId);

×

3314

        char*            in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo);

×

3315

        for (int32_t k = 0; k < pRow->numOfRows; ++k) {

×

3316

          code = colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes);

×

3317

          QUERY_CHECK_CODE(code, lino, _end);

×

3322

    if (pSessionKeys) {

×

3323

      for (int32_t j = 0; j < pRow->numOfRows; ++j) {

×

3324

        const void* px = taosArrayPush(pSessionKeys, pKey);

×

3325

        QUERY_CHECK_NULL(px, code, lino, _end, terrno);

×

3329

    pBlock->info.dataLoad = 1;

×

3330

    pBlock->info.rows += pRow->numOfRows;

×

3332

  code = blockDataUpdateTsWindow(pBlock, 0);

×

3333

  QUERY_CHECK_CODE(code, lino, _end);

×

3335

_end:

×

3336

  if (code != TSDB_CODE_SUCCESS) {

×

3337

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3339

  return code;

×

3342

void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo, SSDataBlock* pBlock,

×

3344

  int32_t        code = TSDB_CODE_SUCCESS;

×

3345

  int32_t        lino = 0;

×

3346

  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

×

3348

  pBlock->info.version = pTaskInfo->version;

×

3350

  blockDataCleanup(pBlock);

×

3351

  taosArrayClear(pSessionKeys);

×

3352

  if (!hasRemainResults(pGroupResInfo)) {

×

3353

    cleanupGroupResInfo(pGroupResInfo);

×

3354

    goto _end;

×

3358

  pBlock->info.id.groupId = 0;

×

3359

  code = buildSessionResultDataBlock(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo, pSessionKeys);

×

3360

  QUERY_CHECK_CODE(code, lino, _end);

×

3362

  if (pBlock->info.rows == 0) {

×

3363

    cleanupGroupResInfo(pGroupResInfo);

×

3366

_end:

×

3367

  if (code != TSDB_CODE_SUCCESS) {

×

3368

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3370

×

3372

static int32_t buildSessionResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

3373

  int32_t                        code = TSDB_CODE_SUCCESS;

×

3374

  int32_t                        lino = 0;

×

3375

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3376

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

3377

  SOptrBasicInfo*                pBInfo = &pInfo->binfo;

×

3378

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

3379

  SStreamNotifyEventSupp*        pNotifySup = &pInfo->basic.notifyEventSup;

×

3380

  STaskNotifyEventStat*          pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;

×

3381

  bool                           addNotifyEvent = false;

×

3382

  addNotifyEvent = IS_NORMAL_SESSION_OP(pOperator) &&

×

3383

                   BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);

×

3384

  doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator, &pInfo->groupResInfo);

×

3385

  if (pInfo->pDelRes->info.rows > 0) {

×

3386

    printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

3387

    if (addNotifyEvent) {

×

3388

      code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);

×

3389

      QUERY_CHECK_CODE(code, lino, _end);

×

3391

    (*ppRes) = pInfo->pDelRes;

×

3392

    return code;

×

3395

  doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes,

×

3397

  if (pBInfo->pRes->info.rows > 0) {

×

3398

    printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

3399

    if (addNotifyEvent) {

×

3401

      for (int32_t i = 0; i < taosArrayGetSize(pNotifySup->pSessionKeys); ++i) {

×

3402

        SSessionKey* pKey = taosArrayGet(pNotifySup->pSessionKeys, i);

×

3403

        pKey->win.ekey += pAggSup->gap;

×

3405

      code = addAggResultNotifyEvent(pBInfo->pRes, pNotifySup->pSessionKeys, pTaskInfo->streamInfo.notifyResultSchema,

×

3407

      QUERY_CHECK_CODE(code, lino, _end);

×

3409

    (*ppRes) = pBInfo->pRes;

×

3410

    return code;

×

3413

  code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);

×

3414

  QUERY_CHECK_CODE(code, lino, _end);

×

3415

  if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {

×

3416

    printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

3417

    (*ppRes) = pNotifySup->pEventBlock;

×

3418

    return code;

×

3421

  code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);

×

3422

  QUERY_CHECK_CODE(code, lino, _end);

×

3424

_end:

×

3425

  if (code != TSDB_CODE_SUCCESS) {

×

3426

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3428

  (*ppRes) = NULL;

×

3429

  return code;

×

3432

int32_t getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins) {

×

3433

  int32_t code = TSDB_CODE_SUCCESS;

×

3434

  int32_t lino = 0;

×

3435

  int32_t size = taosArrayGetSize(pAllWins);

×

3436

  if (size == 0) {

×

3437

    goto _end;

×

3439

  SResultWindowInfo* pWinInfo = taosArrayGet(pAllWins, size - 1);

×

3440

  SSessionKey*       pSeKey = &pWinInfo->sessionWin;

×

3441

  void*              tmp = taosArrayPush(pMaxWins, pSeKey);

×

3442

  if (!tmp) {

×

3443

    code = terrno;

×

3444

    QUERY_CHECK_CODE(code, lino, _end);

×

3447

  if (pSeKey->groupId == 0) {

×

3448

    goto _end;

×

3450

  uint64_t preGpId = pSeKey->groupId;

×

3451

  for (int32_t i = size - 2; i >= 0; i--) {

×

3452

    pWinInfo = taosArrayGet(pAllWins, i);

×

3453

    pSeKey = &pWinInfo->sessionWin;

×

3454

    if (preGpId != pSeKey->groupId) {

×

3455

      void* tmp = taosArrayPush(pMaxWins, pSeKey);

×

3456

      if (!tmp) {

×

3457

        code = terrno;

×

3458

        QUERY_CHECK_CODE(code, lino, _end);

×

3460

      preGpId = pSeKey->groupId;

×

3464

_end:

×

3465

  if (code != TSDB_CODE_SUCCESS) {

×

3466

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3468

  return code;

×

3471

int32_t encodeSSessionKey(void** buf, SSessionKey* key) {

×

3472

  int32_t tlen = 0;

×

3473

  tlen += encodeSTimeWindow(buf, &key->win);

×

3474

  tlen += taosEncodeFixedU64(buf, key->groupId);

×

3475

  return tlen;

×

3478

void* decodeSSessionKey(void* buf, SSessionKey* key) {

×

3479

  buf = decodeSTimeWindow(buf, &key->win);

×

3480

  buf = taosDecodeFixedU64(buf, &key->groupId);

×

3481

  return buf;

×

3484

int32_t encodeSResultWindowInfo(void** buf, SResultWindowInfo* key, int32_t outLen) {

×

3485

  int32_t tlen = 0;

×

3486

  tlen += taosEncodeFixedBool(buf, key->isOutput);

×

3487

  tlen += encodeSSessionKey(buf, &key->sessionWin);

×

3488

  return tlen;

×

3491

void* decodeSResultWindowInfo(void* buf, SResultWindowInfo* key, int32_t outLen) {

×

3492

  buf = taosDecodeFixedBool(buf, &key->isOutput);

×

3493

  buf = decodeSSessionKey(buf, &key->sessionWin);

×

3494

  return buf;

×

3497

int32_t doStreamSessionEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOperator, bool isParent) {

×

3498

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3499

  if (!pInfo) {

×

3500

    return 0;

×

3503

  void* pData = (buf == NULL) ? NULL : *buf;

×

3506

  int32_t tlen = 0;

×

3507

  int32_t mapSize = tSimpleHashGetSize(pInfo->streamAggSup.pResultRows);

×

3508

  tlen += taosEncodeFixedI32(buf, mapSize);

×

3509

  void*   pIte = NULL;

×

3510

  size_t  keyLen = 0;

×

3511

  int32_t iter = 0;

×

3512

  while ((pIte = tSimpleHashIterate(pInfo->streamAggSup.pResultRows, pIte, &iter)) != NULL) {

×

3513

    void* key = tSimpleHashGetKey(pIte, &keyLen);

×

3514

    tlen += encodeSSessionKey(buf, key);

×

3515

    tlen += encodeSResultWindowInfo(buf, pIte, pInfo->streamAggSup.resultRowSize);

×

3519

  tlen += encodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);

×

3522

  int32_t size = taosArrayGetSize(pInfo->pChildren);

×

3523

  tlen += taosEncodeFixedI32(buf, size);

×

3524

  for (int32_t i = 0; i < size; i++) {

×

3525

    SOperatorInfo* pChOp = taosArrayGetP(pInfo->pChildren, i);

×

3526

    tlen += doStreamSessionEncodeOpState(buf, 0, pChOp, false);

×

3530

  tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);

×

3533

  tlen += encodeStreamBasicInfo(buf, &pInfo->basic);

×

3536

  if (isParent) {

×

3537

    if (buf) {

×

3538

      uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));

×

3539

      tlen += taosEncodeFixedU32(buf, cksum);

×

3541

      tlen += sizeof(uint32_t);

×

3545

  return tlen;

×

3548

int32_t doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator, bool isParent, void** ppBuf) {

×

3549

  int32_t                        code = TSDB_CODE_SUCCESS;

×

3550

  int32_t                        lino = 0;

×

3551

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3552

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

3553

  void*                        pDataEnd = POINTER_SHIFT(buf, len);

×

3554

  if (!pInfo) {

×

3555

    code = TSDB_CODE_FAILED;

×

3556

    QUERY_CHECK_CODE(code, lino, _end);

×

3558

  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;

×

3561

  if (isParent) {

×

3562

    int32_t dataLen = len - sizeof(uint32_t);

×

3563

    void*   pCksum = POINTER_SHIFT(buf, dataLen);

×

3564

    if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) {

×

3565

      qError("stream session state is invalid");

×

3566

      code = TSDB_CODE_FAILED;

×

3567

      QUERY_CHECK_CODE(code, lino, _end);

×

3569

    pDataEnd = pCksum;

×

3573

  int32_t mapSize = 0;

×

3574

  buf = taosDecodeFixedI32(buf, &mapSize);

×

3575

  for (int32_t i = 0; i < mapSize; i++) {

×

3576

    SResultWindowInfo winfo = {0};

×

3577

    buf = decodeSSessionKey(buf, &winfo.sessionWin);

×

3578

    int32_t winCode = TSDB_CODE_SUCCESS;

×

3579

    code = pAggSup->stateStore.streamStateSessionAddIfNotExist(

×

3581

    QUERY_CHECK_CODE(code, lino, _end);

×

3582

    QUERY_CHECK_CONDITION((winCode == TSDB_CODE_SUCCESS), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);

×

3584

    buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize);

×

3585

    code = tSimpleHashPut(pInfo->streamAggSup.pResultRows, &winfo.sessionWin, sizeof(SSessionKey), &winfo,

×

3587

    QUERY_CHECK_CODE(code, lino, _end);

×

3591

  buf = decodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);

×

3594

  int32_t size = 0;

×

3595

  buf = taosDecodeFixedI32(buf, &size);

×

3596

  for (int32_t i = 0; i < size; i++) {

×

3597

    SOperatorInfo* pChOp = taosArrayGetP(pInfo->pChildren, i);

×

3598

    code = doStreamSessionDecodeOpState(buf, 0, pChOp, false, &buf);

×

3599

    QUERY_CHECK_CODE(code, lino, _end);

×

3603

  buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);

×

3604

  if (ppBuf) {

×

3605

    (*ppBuf) = buf;

×

3609

  if (buf < pDataEnd) {

×

3610

    code = decodeStreamBasicInfo(&buf, &pInfo->basic);

×

3611

    QUERY_CHECK_CODE(code, lino, _end);

×

3614

_end:

×

3615

  if (code != TSDB_CODE_SUCCESS) {

×

3616

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3618

  return code;

×

3621

void doStreamSessionSaveCheckpoint(SOperatorInfo* pOperator) {

×

3622

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3623

  if (needSaveStreamOperatorInfo(&pInfo->basic)) {

×

3624

    int32_t len = doStreamSessionEncodeOpState(NULL, 0, pOperator, true);

×

3625

    void*   buf = taosMemoryCalloc(1, len);

×

3626

    if (!buf) {

×

3627

      qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY));

×

3628

      return;

×

3630

    void* pBuf = buf;

×

3631

    len = doStreamSessionEncodeOpState(&pBuf, len, pOperator, true);

×

3632

    pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME,

×

3634

    taosMemoryFree(buf);

×

3635

    saveStreamOperatorStateComplete(&pInfo->basic);

×

3639

void resetUnCloseSessionWinInfo(SSHashObj* winMap) {

×

3640

  void*   pIte = NULL;

×

3641

  int32_t iter = 0;

×

3642

  while ((pIte = tSimpleHashIterate(winMap, pIte, &iter)) != NULL) {

×

3643

    SResultWindowInfo* pResInfo = pIte;

×

3644

    pResInfo->pStatePos->beUsed = true;

×

3646

×

3648

int32_t copyDeleteSessionKey(SSHashObj* source, SSHashObj* dest) {

×

3649

  int32_t code = TSDB_CODE_SUCCESS;

×

3650

  int32_t lino = 0;

×

3651

  if (tSimpleHashGetSize(source) == 0) {

×

3652

    goto _end;

×

3654

  void*   pIte = NULL;

×

3655

  int32_t iter = 0;

×

3656

  size_t  keyLen = 0;

×

3657

  while ((pIte = tSimpleHashIterate(source, pIte, &iter)) != NULL) {

×

3658

    SSessionKey* pKey = tSimpleHashGetKey(pIte, &keyLen);

×

3659

    code = saveDeleteRes(dest, *pKey);

×

3660

    QUERY_CHECK_CODE(code, lino, _end);

×

3662

  tSimpleHashClear(source);

×

3664

_end:

×

3665

  if (code != TSDB_CODE_SUCCESS) {

×

3666

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3668

  return code;

×

3671

static int32_t doStreamSessionAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

3672

  int32_t                        code = TSDB_CODE_SUCCESS;

×

3673

  int32_t                        lino = 0;

×

3674

  SExprSupp*                     pSup = &pOperator->exprSupp;

×

3675

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3676

  SOptrBasicInfo*                pBInfo = &pInfo->binfo;

×

3677

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

3678

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

3679

  qDebug("stask:%s  %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);

×

3680

  if (pOperator->status == OP_EXEC_DONE) {

×

3681

    (*ppRes) = NULL;

×

3682

    return code;

×

3683

  } else if (pOperator->status == OP_RES_TO_RETURN) {

×

3684

    SSDataBlock* opRes = NULL;

×

3685

    code = buildSessionResult(pOperator, &opRes);

×

3686

    QUERY_CHECK_CODE(code, lino, _end);

×

3687

    if (opRes) {

×

3688

      (*ppRes) = opRes;

×

3689

      return code;

×

3692

    if (pInfo->recvGetAll) {

×

3693

      pInfo->recvGetAll = false;

×

3694

      resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows);

×

3697

    if (pInfo->reCkBlock) {

×

3698

      pInfo->reCkBlock = false;

×

3699

      printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

3700

      (*ppRes) = pInfo->pCheckpointRes;

×

3701

      return code;

×

3704

    setStreamOperatorCompleted(pOperator);

×

3705

    (*ppRes) = NULL;

×

3706

    return code;

×

3709

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

3710

  if (!pInfo->pUpdated) {

×

3711

    pInfo->pUpdated = taosArrayInit(16, sizeof(SResultWindowInfo));

×

3712

    QUERY_CHECK_NULL(pInfo->pUpdated, code, lino, _end, terrno);

×

3714

  if (!pInfo->pStUpdated) {

×

3715

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

3716

    pInfo->pStUpdated = tSimpleHashInit(64, hashFn);

×

3717

    QUERY_CHECK_NULL(pInfo->pStUpdated, code, lino, _end, terrno);

×

3719

  while (1) {

×

3720

    SSDataBlock* pBlock = NULL;

×

3721

    code = downstream->fpSet.getNextFn(downstream, &pBlock);

×

3722

    QUERY_CHECK_CODE(code, lino, _end);

×

3724

    if (pBlock == NULL) {

×

3725

      break;

×

3727

    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));

×

3728

    setStreamOperatorState(&pInfo->basic, pBlock->info.type);

×

3730

    if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||

×

3731

        pBlock->info.type == STREAM_CLEAR) {

×

3732

      SArray* pWins = taosArrayInit(16, sizeof(SSessionKey));

×

3733

      QUERY_CHECK_NULL(pWins, code, lino, _end, terrno);

×

3735

      code = doDeleteTimeWindows(pAggSup, pBlock, pWins);

×

3736

      QUERY_CHECK_CODE(code, lino, _end);

×

3738

      removeSessionResults(pAggSup, pInfo->pStUpdated, pWins);

×

3739

      if (IS_FINAL_SESSION_OP(pOperator)) {

×

3740

        int32_t                        childIndex = getChildIndex(pBlock);

×

3741

        SOperatorInfo*                 pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);

×

3742

        SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;

×

3744

        code = doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, NULL);

×

3745

        QUERY_CHECK_CODE(code, lino, _end);

×

3747

        code = rebuildSessionWindow(pOperator, pWins, pInfo->pStUpdated);

×

3748

        QUERY_CHECK_CODE(code, lino, _end);

×

3750

      code = copyDeleteWindowInfo(pWins, pInfo->pStDeleted);

×

3751

      QUERY_CHECK_CODE(code, lino, _end);

×

3753

      if (pInfo->destHasPrimaryKey && IS_NORMAL_SESSION_OP(pOperator)) {

×

3754

        code = copyDeleteWindowInfo(pWins, pInfo->pPkDeleted);

×

3755

        QUERY_CHECK_CODE(code, lino, _end);

×

3757

      taosArrayDestroy(pWins);

×

3758

      continue;

×

3759

    } else if (pBlock->info.type == STREAM_GET_ALL) {

×

3760

      pInfo->recvGetAll = true;

×

3761

      code = getAllSessionWindow(pAggSup->pResultRows, pInfo->pStUpdated);

×

3762

      QUERY_CHECK_CODE(code, lino, _end);

×

3763

      continue;

×

3764

    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {

×

3765

      (*ppRes) = pBlock;

×

3766

      return code;

×

3767

    } else if (pBlock->info.type == STREAM_CHECKPOINT) {

×

3768

      pAggSup->stateStore.streamStateCommit(pAggSup->pState);

×

3769

      doStreamSessionSaveCheckpoint(pOperator);

×

3770

      code = copyDataBlock(pInfo->pCheckpointRes, pBlock);

×

3771

      QUERY_CHECK_CODE(code, lino, _end);

×

3773

      continue;

×

3775

      if (pBlock->info.type != STREAM_NORMAL && pBlock->info.type != STREAM_INVALID) {

×

3776

        code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;

×

3777

        QUERY_CHECK_CODE(code, lino, _end);

×

3781

    if (pInfo->scalarSupp.pExprInfo != NULL) {

×

3782

      SExprSupp* pExprSup = &pInfo->scalarSupp;

×

3783

      code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);

×

3784

      QUERY_CHECK_CODE(code, lino, _end);

×

3787

    code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

3788

    QUERY_CHECK_CODE(code, lino, _end);

×

3790

    doStreamSessionAggImpl(pOperator, pBlock, pInfo->pStUpdated, pInfo->pStDeleted, IS_FINAL_SESSION_OP(pOperator),

×

3792

    if (IS_FINAL_SESSION_OP(pOperator)) {

×

3793

      int32_t chIndex = getChildIndex(pBlock);

×

3794

      int32_t size = taosArrayGetSize(pInfo->pChildren);

×

3796

      for (int32_t i = 0; i < chIndex + 1 - size; i++) {

×

3797

        SOperatorInfo* pChildOp = NULL;

×

3798

        code = createStreamFinalSessionAggOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0, NULL, &pChildOp);

×

3799

        if (pChildOp == NULL || code != 0) {

×

3800

          qError("%s create stream child of final session error", GET_TASKID(pTaskInfo));

×

3801

          code = TSDB_CODE_FAILED;

×

3802

          QUERY_CHECK_CODE(code, lino, _end);

×

3805

        void* tmp = taosArrayPush(pInfo->pChildren, &pChildOp);

×

3806

        if (!tmp) {

×

3807

          code = terrno;

×

3808

          QUERY_CHECK_CODE(code, lino, _end);

×

3812

      SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex);

×

3813

      code = setInputDataBlock(&pChildOp->exprSupp, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

3814

      QUERY_CHECK_CODE(code, lino, _end);

×

3815

      doStreamSessionAggImpl(pChildOp, pBlock, NULL, NULL, true, false);

×

3817

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);

×

3818

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.watermark);

×

3821

  pOperator->status = OP_RES_TO_RETURN;

×

3823

  code = closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pInfo->pStUpdated);

×

3824

  QUERY_CHECK_CODE(code, lino, _end);

×

3826

  code = closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs);

×

3827

  QUERY_CHECK_CODE(code, lino, _end);

×

3829

  code = copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc);

×

3830

  QUERY_CHECK_CODE(code, lino, _end);

×

3832

  if (!pInfo->destHasPrimaryKey) {

×

3833

    removeSessionDeleteResults(pInfo->pStDeleted, pInfo->pUpdated);

×

3835

  if (pInfo->isHistoryOp) {

×

3836

    code = getMaxTsWins(pInfo->pUpdated, pInfo->historyWins);

×

3837

    QUERY_CHECK_CODE(code, lino, _end);

×

3839

  if (pInfo->destHasPrimaryKey && IS_NORMAL_SESSION_OP(pOperator)) {

×

3840

    code = copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pStDeleted);

×

3841

    QUERY_CHECK_CODE(code, lino, _end);

×

3843

  initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);

×

3844

  pInfo->pUpdated = NULL;

×

3845

  code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);

×

3846

  QUERY_CHECK_CODE(code, lino, _end);

×

3848

  SSDataBlock* opRes = NULL;

×

3849

  code = buildSessionResult(pOperator, &opRes);

×

3850

  QUERY_CHECK_CODE(code, lino, _end);

×

3851

  if (opRes) {

×

3852

    (*ppRes) = opRes;

×

3853

    return code;

×

3856

_end:

×

3857

  if (code != TSDB_CODE_SUCCESS) {

×

3858

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3859

    pTaskInfo->code = code;

×

3860

    T_LONG_JMP(pTaskInfo->env, code);

×

3862

  setStreamOperatorCompleted(pOperator);

×

3863

  (*ppRes) = NULL;

×

3864

  return code;

×

3867

static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {

×

3868

  SSDataBlock* pRes = NULL;

×

3869

  int32_t      code = doStreamSessionAggNext(pOperator, &pRes);

×

3870

  return pRes;

×

3873

void streamSessionReleaseState(SOperatorInfo* pOperator) {

×

3874

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3875

  int32_t                        winSize = taosArrayGetSize(pInfo->historyWins) * sizeof(SSessionKey);

×

3876

  int32_t                        resSize = winSize + sizeof(TSKEY);

×

3877

  char*                          pBuff = taosMemoryCalloc(1, resSize);

×

3878

  if (!pBuff) {

×

3879

    return;

×

3881

  memcpy(pBuff, pInfo->historyWins->pData, winSize);

×

3882

  memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY));

×

3883

  pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_STATE_NAME,

×

3885

  pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState);

×

3886

  taosMemoryFreeClear(pBuff);

×

3887

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

3888

  if (downstream->fpSet.releaseStreamStateFn) {

×

3889

    downstream->fpSet.releaseStreamStateFn(downstream);

×

3893

void resetWinRange(STimeWindow* winRange) {

×

3894

  winRange->skey = INT64_MIN;

×

3895

  winRange->ekey = INT64_MAX;

×

3896

×

3898

int32_t getSessionWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SResultWindowInfo* pWinInfo) {

×

3899

  int32_t code = TSDB_CODE_SUCCESS;

×

3900

  int32_t lino = 0;

×

3901

  int32_t rowSize = pAggSup->resultRowSize;

×

3902

  int32_t winCode = TSDB_CODE_SUCCESS;

×

3903

  code = pAggSup->stateStore.streamStateSessionGet(pAggSup->pState, pKey, (void**)&pWinInfo->pStatePos, &rowSize,

×

3905

  QUERY_CHECK_CODE(code, lino, _end);

×

3907

  if (winCode == TSDB_CODE_SUCCESS) {

×

3908

    pWinInfo->sessionWin = *pKey;

×

3909

    pWinInfo->isOutput = true;

×

3910

    if (pWinInfo->pStatePos->needFree) {

×

3911

      pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pWinInfo->sessionWin);

×

3914

    SET_SESSION_WIN_INVALID((*pWinInfo));

×

3917

_end:

×

3918

  if (code != TSDB_CODE_SUCCESS) {

×

3919

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3921

  return code;

×

3924

void reloadAggSupFromDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup) {

×

3925

  SStateStore* pAPI = &downstream->pTaskInfo->storageAPI.stateStore;

×

3927

  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {

×

3928

    reloadAggSupFromDownStream(downstream->pDownstream[0], pAggSup);

×

3929

    return;

×

3932

  SStreamScanInfo* pScanInfo = downstream->info;

×

3933

  pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo;

×

3936

void streamSessionSemiReloadState(SOperatorInfo* pOperator) {

×

3937

  int32_t                        code = TSDB_CODE_SUCCESS;

×

3938

  int32_t                        lino = 0;

×

3939

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3940

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

3941

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

3942

  resetWinRange(&pAggSup->winRange);

×

3944

  SResultWindowInfo winInfo = {0};

×

3945

  int32_t           size = 0;

×

3946

  void*             pBuf = NULL;

×

3947

  code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME,

×

3949

  QUERY_CHECK_CODE(code, lino, _end);

×

3951

  int32_t      num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);

×

3952

  SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;

×

3953

  for (int32_t i = 0; i < num; i++) {

×

3954

    SResultWindowInfo winInfo = {0};

×

3955

    code = getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &winInfo);

×

3956

    QUERY_CHECK_CODE(code, lino, _end);

×

3957

    if (!IS_VALID_SESSION_WIN(winInfo)) {

×

3958

      continue;

×

3960

    compactSessionSemiWindow(pOperator, &winInfo);

×

3961

    code = saveSessionOutputBuf(pAggSup, &winInfo);

×

3962

    QUERY_CHECK_CODE(code, lino, _end);

×

3964

  TSKEY ts = *(TSKEY*)((char*)pBuf + size - sizeof(TSKEY));

×

3965

  taosMemoryFree(pBuf);

×

3966

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);

×

3967

  pAggSup->stateStore.streamStateReloadInfo(pAggSup->pState, ts);

×

3969

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

3970

  if (downstream->fpSet.reloadStreamStateFn) {

×

3971

    downstream->fpSet.reloadStreamStateFn(downstream);

×

3973

  reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup);

×

3975

_end:

×

3976

  if (code != TSDB_CODE_SUCCESS) {

×

3977

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3979

×

3981

void streamSessionReloadState(SOperatorInfo* pOperator) {

×

3982

  int32_t                        code = TSDB_CODE_SUCCESS;

×

3983

  int32_t                        lino = 0;

×

3984

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3985

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

3986

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

3987

  resetWinRange(&pAggSup->winRange);

×

3989

  int32_t size = 0;

×

3990

  void*   pBuf = NULL;

×

3991

  code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME,

×

3994

  QUERY_CHECK_CODE(code, lino, _end);

×

3996

  int32_t      num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);

×

3997

  SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;

×

3999

  TSKEY ts = *(TSKEY*)((char*)pBuf + size - sizeof(TSKEY));

×

4000

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);

×

4001

  pAggSup->stateStore.streamStateReloadInfo(pAggSup->pState, ts);

×

4003

  if (!pInfo->pStUpdated && num > 0) {

×

4004

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

4005

    pInfo->pStUpdated = tSimpleHashInit(64, hashFn);

×

4006

    QUERY_CHECK_NULL(pInfo->pStUpdated, code, lino, _end, terrno);

×

4008

  for (int32_t i = 0; i < num; i++) {

×

4009

    SResultWindowInfo winInfo = {0};

×

4010

    code = getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &winInfo);

×

4011

    QUERY_CHECK_CODE(code, lino, _end);

×

4012

    if (!IS_VALID_SESSION_WIN(winInfo)) {

×

4013

      continue;

×

4016

    int32_t winNum = 0;

×

4017

    code = compactSessionWindow(pOperator, &winInfo, pInfo->pStUpdated, pInfo->pStDeleted, true, &winNum, NULL);

×

4018

    QUERY_CHECK_CODE(code, lino, _end);

×

4020

    if (winNum > 0) {

×

4021

      qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, winInfo.sessionWin.win.skey,

×

4023

      if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {

×

4024

        code = saveResult(winInfo, pInfo->pStUpdated);

×

4025

        QUERY_CHECK_CODE(code, lino, _end);

×

4026

      } else if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

4027

        if (!isCloseWindow(&winInfo.sessionWin.win, &pInfo->twAggSup)) {

×

4028

          code = saveDeleteRes(pInfo->pStDeleted, winInfo.sessionWin);

×

4029

          QUERY_CHECK_CODE(code, lino, _end);

×

4031

        SSessionKey key = {0};

×

4032

        getSessionHashKey(&winInfo.sessionWin, &key);

×

4033

        code = tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo));

×

4034

        QUERY_CHECK_CODE(code, lino, _end);

×

4037

    code = saveSessionOutputBuf(pAggSup, &winInfo);

×

4038

    QUERY_CHECK_CODE(code, lino, _end);

×

4040

  taosMemoryFree(pBuf);

×

4042

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

4043

  if (downstream->fpSet.reloadStreamStateFn) {

×

4044

    downstream->fpSet.reloadStreamStateFn(downstream);

×

4046

  reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup);

×

4048

_end:

×

4049

  if (code != TSDB_CODE_SUCCESS) {

×

4050

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

4052

×

4054

int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,

×

4056

  QRY_PARAM_CHECK(pOptrInfo);

×

4058

  SSessionWinodwPhysiNode*       pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;

×

4059

  int32_t                        numOfCols = 0;

×

4060

  int32_t                        code = TSDB_CODE_OUT_OF_MEMORY;

×

4061

  int32_t                        lino = 0;

×

4062

  SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo));

×

4063

  SOperatorInfo*                 pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));

×

4064

  if (pInfo == NULL || pOperator == NULL) {

×

4065

    code = terrno;

×

4066

    goto _error;

×

4069

  pOperator->pTaskInfo = pTaskInfo;

×

4071

  initResultSizeInfo(&pOperator->resultInfo, 4096);

×

4072

  if (pSessionNode->window.pExprs != NULL) {

×

4073

    int32_t    numOfScalar = 0;

×

4074

    SExprInfo* pScalarExprInfo = NULL;

×

4075

    code = createExprInfo(pSessionNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalar);

×

4076

    QUERY_CHECK_CODE(code, lino, _error);

×

4078

    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);

×

4079

    if (code != TSDB_CODE_SUCCESS) {

×

4080

      goto _error;

×

4083

  SExprSupp* pExpSup = &pOperator->exprSupp;

×

4085

  SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);

×

4086

  QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);

×

4087

  pInfo->binfo.pRes = pResBlock;

×

4089

  SExprInfo* pExprInfo = NULL;

×

4090

  code = createExprInfo(pSessionNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);

×

4091

  QUERY_CHECK_CODE(code, lino, _error);

×

4093

  code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore);

×

4094

  QUERY_CHECK_CODE(code, lino, _error);

×

4096

  pInfo->twAggSup = (STimeWindowAggSupp){

×

4097

      .waterMark = pSessionNode->window.watermark,

×

4098

      .calTrigger = pSessionNode->window.triggerType,

×

4101

      .deleteMark = getDeleteMark(&pSessionNode->window, 0),

×

4104

  pInfo->primaryTsIndex = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;

×

4106

      initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, pSessionNode->gap, pTaskInfo->streamInfo.pState,

×

4107

                             0, 0, &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup, GET_TASKID(pTaskInfo),

×

4109

  if (code != TSDB_CODE_SUCCESS) {

×

4110

    goto _error;

×

4113

  code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

×

4114

  QUERY_CHECK_CODE(code, lino, _error);

×

4116

  if (pSessionNode->window.pTsEnd) {

×

4117

    pInfo->endTsIndex = ((SColumnNode*)pSessionNode->window.pTsEnd)->slotId;

×

4120

  pInfo->order = TSDB_ORDER_ASC;

×

4121

  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

4122

  pInfo->pStDeleted = tSimpleHashInit(64, hashFn);

×

4123

  QUERY_CHECK_NULL(pInfo->pStDeleted, code, lino, _error, terrno);

×

4124

  pInfo->pDelIterator = NULL;

×

4125

  code = createSpecialDataBlock(STREAM_DELETE_RESULT, &pInfo->pDelRes);

×

4126

  QUERY_CHECK_CODE(code, lino, _error);

×

4128

  pInfo->pChildren = NULL;

×

4129

  pInfo->pPhyNode = pPhyNode;

×

4130

  pInfo->ignoreExpiredData = pSessionNode->window.igExpired;

×

4131

  pInfo->ignoreExpiredDataSaved = false;

×

4132

  pInfo->pUpdated = NULL;

×

4133

  pInfo->pStUpdated = NULL;

×

4134

  pInfo->dataVersion = 0;

×

4135

  pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey));

×

4136

  if (!pInfo->historyWins) {

×

4137

    goto _error;

×

4139

  if (pHandle) {

×

4140

    pInfo->isHistoryOp = (pHandle->fillHistory == STREAM_HISTORY_OPERATOR);

×

4143

  code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);

×

4144

  QUERY_CHECK_CODE(code, lino, _error);

×

4146

  pInfo->clearState = false;

×

4147

  pInfo->recvGetAll = false;

×

4148

  pInfo->destHasPrimaryKey = pSessionNode->window.destHasPrimaryKey;

×

4149

  pInfo->pPkDeleted = tSimpleHashInit(64, hashFn);

×

4150

  QUERY_CHECK_NULL(pInfo->pPkDeleted, code, lino, _error, terrno);

×

4151

  pInfo->pOperator = pOperator;

×

4152

  initNonBlockAggSupptor(&pInfo->nbSup, NULL, NULL);

×

4154

  setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), nodeType(pSessionNode), true,

×

4156

  if (pPhyNode->type != QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {

×

4158

    void*   buff = NULL;

×

4159

    int32_t len = 0;

×

4161

        pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME,

×

4163

    if (res == TSDB_CODE_SUCCESS) {

×

4164

      code = doStreamSessionDecodeOpState(buff, len, pOperator, true, NULL);

×

4165

      taosMemoryFree(buff);

×

4166

      QUERY_CHECK_CODE(code, lino, _error);

×

4170

  code = initStreamBasicInfo(&pInfo->basic, pOperator);

×

4171

  QUERY_CHECK_CODE(code, lino, _error);

×

4173

  if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {

×

4174

    if (pHandle->fillHistory == STREAM_HISTORY_OPERATOR) {

×

4175

      setFillHistoryOperatorFlag(&pInfo->basic);

×

4176

    } else if (pHandle->fillHistory == STREAM_RECALCUL_OPERATOR) {

×

4177

      setRecalculateOperatorFlag(&pInfo->basic);

×

4180

        createOperatorFpSet(optrDummyOpenFn, doStreamSessionNonblockAggNext, NULL, destroyStreamSessionAggOperatorInfo,

×

4182

    setOperatorStreamStateFn(pOperator, streamSessionNonblockReleaseState, streamSessionNonblockReloadState);

×

4185

        createOperatorFpSet(optrDummyOpenFn, doStreamSessionAggNext, NULL, destroyStreamSessionAggOperatorInfo,

×

4187

    setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState);

×

4190

  if (downstream) {

×

4191

    pInfo->basic.primaryPkIndex = -1;

×

4192

    code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,

×

4193

                          &pInfo->twAggSup, &pInfo->basic, pSessionNode->window.recalculateInterval);

×

4194

    QUERY_CHECK_CODE(code, lino, _error);

×

4196

    code = appendDownstream(pOperator, &downstream, 1);

×

4197

    QUERY_CHECK_CODE(code, lino, _error);

×

4200

  *pOptrInfo = pOperator;

×

4201

  return TSDB_CODE_SUCCESS;

×

4203

_error:

×

4204

  if (pInfo != NULL) {

×

4205

    destroyStreamSessionAggOperatorInfo(pInfo);

×

4207

  destroyOperatorAndDownstreams(pOperator, &downstream, 1);

×

4208

  pTaskInfo->code = code;

×

4209

  qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

4210

  return code;

×

4213

static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) {

×

4214

  tSimpleHashClear(pInfo->streamAggSup.pResultRows);

×

4215

  pInfo->streamAggSup.stateStore.streamStateSessionClear(pInfo->streamAggSup.pState);

×

4216

  pInfo->clearState = false;

×

4217

×

4219

int32_t deleteSessionWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate,

×

4221

  int32_t code = TSDB_CODE_SUCCESS;

×

4222

  int32_t lino = 0;

×

4223

  SArray* pWins = taosArrayInit(16, sizeof(SSessionKey));

×

4224

  if (!pWins) {

×

4225

    code = terrno;

×

4226

    QUERY_CHECK_CODE(code, lino, _end);

×

4228

  code = doDeleteTimeWindows(pAggSup, pBlock, pWins);

×

4229

  QUERY_CHECK_CODE(code, lino, _end);

×

4231

  removeSessionResults(pAggSup, pMapUpdate, pWins);

×

4232

  code = copyDeleteWindowInfo(pWins, pMapDelete);

×

4233

  QUERY_CHECK_CODE(code, lino, _end);

×

4235

  if (needAdd) {

×

4236

    code = copyDeleteWindowInfo(pWins, pPkDelete);

×

4237

    QUERY_CHECK_CODE(code, lino, _end);

×

4239

  taosArrayDestroy(pWins);

×

4241

_end:

×

4242

  if (code != TSDB_CODE_SUCCESS) {

×

4243

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

4245

  return code;

×

4248

static int32_t doStreamSessionSemiAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

4249

  int32_t                        code = TSDB_CODE_SUCCESS;

×

4250

  int32_t                        lino = 0;

×

4251

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

4252

  SOptrBasicInfo*                pBInfo = &pInfo->binfo;

×

4253

  TSKEY                          maxTs = INT64_MIN;

×

4254

  SExprSupp*                     pSup = &pOperator->exprSupp;

×

4255

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

4256

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

4258

  qDebug("stask:%s  %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);

×

4259

  if (pOperator->status == OP_EXEC_DONE) {

×

4260

    (*ppRes) = NULL;

×

4261

    return code;

×

4265

    SSDataBlock* opRes = NULL;

×

4266

    code = buildSessionResult(pOperator, &opRes);

×

4267

    QUERY_CHECK_CODE(code, lino, _end);

×

4268

    if (opRes) {

×

4269

      (*ppRes) = opRes;

×

4270

      return code;

×

4273

    if (pInfo->clearState) {

×

4274

      clearFunctionContext(&pOperator->exprSupp);

×

4276

      clearStreamSessionOperator(pInfo);

×

4279

    if (pOperator->status == OP_RES_TO_RETURN) {

×

4280

      if (pInfo->reCkBlock) {

×

4281

        pInfo->reCkBlock = false;

×

4282

        printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

4283

        (*ppRes) = pInfo->pCheckpointRes;

×

4284

        return code;

×

4286

      clearFunctionContext(&pOperator->exprSupp);

×

4288

      clearStreamSessionOperator(pInfo);

×

4289

      setStreamOperatorCompleted(pOperator);

×

4290

      (*ppRes) = NULL;

×

4291

      return code;

×

4295

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

4296

  if (!pInfo->pUpdated) {

×

4297

    pInfo->pUpdated = taosArrayInit(16, sizeof(SResultWindowInfo));

×

4298

    QUERY_CHECK_NULL(pInfo->pUpdated, code, lino, _end, terrno);

×

4300

  if (!pInfo->pStUpdated) {

×

4301

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

4302

    pInfo->pStUpdated = tSimpleHashInit(64, hashFn);

×

4303

    QUERY_CHECK_NULL(pInfo->pStUpdated, code, lino, _end, terrno);

×

4305

  while (1) {

×

4306

    SSDataBlock* pBlock = NULL;

×

4307

    code = downstream->fpSet.getNextFn(downstream, &pBlock);

×

4308

    QUERY_CHECK_CODE(code, lino, _end);

×

4310

    if (pBlock == NULL) {

×

4311

      pOperator->status = OP_RES_TO_RETURN;

×

4312

      break;

×

4314

    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));

×

4315

    setStreamOperatorState(&pInfo->basic, pBlock->info.type);

×

4317

    if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||

×

4318

        pBlock->info.type == STREAM_CLEAR) {

×

4320

      code = deleteSessionWinState(pAggSup, pBlock, pInfo->pStUpdated, pInfo->pStDeleted, NULL, false);

×

4321

      QUERY_CHECK_CODE(code, lino, _end);

×

4322

      pInfo->clearState = true;

×

4323

      break;

×

4324

    } else if (pBlock->info.type == STREAM_GET_ALL) {

×

4325

      code = getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pStUpdated);

×

4326

      QUERY_CHECK_CODE(code, lino, _end);

×

4327

      continue;

×

4328

    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {

×

4329

      (*ppRes) = pBlock;

×

4330

      return code;

×

4331

    } else if (pBlock->info.type == STREAM_CHECKPOINT) {

×

4332

      pAggSup->stateStore.streamStateCommit(pAggSup->pState);

×

4333

      doStreamSessionSaveCheckpoint(pOperator);

×

4334

      continue;

×

4336

      if (pBlock->info.type != STREAM_NORMAL && pBlock->info.type != STREAM_INVALID) {

×

4337

        code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;

×

4338

        QUERY_CHECK_CODE(code, lino, _end);

×

4342

    if (pInfo->scalarSupp.pExprInfo != NULL) {

×

4343

      SExprSupp* pExprSup = &pInfo->scalarSupp;

×

4344

      code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);

×

4345

      QUERY_CHECK_CODE(code, lino, _end);

×

4348

    code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

4349

    QUERY_CHECK_CODE(code, lino, _end);

×

4350

    doStreamSessionAggImpl(pOperator, pBlock, pInfo->pStUpdated, NULL, false, false);

×

4351

    maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);

×

4354

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);

×

4355

  pBInfo->pRes->info.watermark = pInfo->twAggSup.maxTs;

×

4357

  code = copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc);

×

4358

  QUERY_CHECK_CODE(code, lino, _end);

×

4360

  removeSessionDeleteResults(pInfo->pStDeleted, pInfo->pUpdated);

×

4362

  if (pInfo->isHistoryOp) {

×

4363

    code = getMaxTsWins(pInfo->pUpdated, pInfo->historyWins);

×

4364

    QUERY_CHECK_CODE(code, lino, _end);

×

4367

  initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);

×

4368

  pInfo->pUpdated = NULL;

×

4369

  code = blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);

×

4370

  QUERY_CHECK_CODE(code, lino, _end);

×

4372

  SSDataBlock* opRes = NULL;

×

4373

  code = buildSessionResult(pOperator, &opRes);

×

4374

  QUERY_CHECK_CODE(code, lino, _end);

×

4375

  if (opRes) {

×

4376

    (*ppRes) = opRes;

×

4377

    return code;

×

4380

_end:

×

4381

  if (code != TSDB_CODE_SUCCESS) {

×

4382

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

4383

    pTaskInfo->code = code;

×

4384

    T_LONG_JMP(pTaskInfo->env, code);

×

4387

  clearFunctionContext(&pOperator->exprSupp);

×

4389

  clearStreamSessionOperator(pInfo);

×

4390

  setStreamOperatorCompleted(pOperator);

×

4391

  (*ppRes) = NULL;

×

4392

  return code;

×

4395

static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {

×

4396

  SSDataBlock* pRes = NULL;

×

4397

  int32_t      code = doStreamSessionSemiAggNext(pOperator, &pRes);

×

4398

  return pRes;

×

4401

int32_t createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,

×

4404

  QRY_PARAM_CHECK(pOptrInfo);

×

4406

  int32_t        code = TSDB_CODE_SUCCESS;

×

4407

  int32_t        lino = 0;

×

4408

  SOperatorInfo* pOperator = NULL;

×

4409

  code = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo, pHandle, &pOperator);

×

4410

  if (pOperator == NULL || code != 0) {

×

4411

    downstream = NULL;

×

4412

    QUERY_CHECK_CODE(code, lino, _error);

×

4415

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

4416

  pInfo->pOperator = pOperator;

×

4418

  if (pPhyNode->type != QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {

×

4419

    pOperator->fpSet =

×

4420

        createOperatorFpSet(optrDummyOpenFn, doStreamSessionSemiAggNext, NULL, destroyStreamSessionAggOperatorInfo,

×

4422

    setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionSemiReloadState);

×

4425

  if (numOfChild > 0) {

×

4426

    pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*));

×

4427

    QUERY_CHECK_NULL(pInfo->pChildren, code, lino, _error, terrno);

×

4428

    for (int32_t i = 0; i < numOfChild; i++) {

×

4429

      SOperatorInfo* pChildOp = NULL;

×

4430

      code = createStreamFinalSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo, 0, pHandle, &pChildOp);

×

4431

      if (pChildOp == NULL || code != 0) {

×

4432

        QUERY_CHECK_CODE(code, lino, _error);

×

4435

      SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info;

×

4436

      pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;

×

4437

      pTaskInfo->storageAPI.stateStore.streamStateSetNumber(pChInfo->streamAggSup.pState, i, pInfo->primaryTsIndex);

×

4438

      void* tmp = taosArrayPush(pInfo->pChildren, &pChildOp);

×

4439

      if (!tmp) {

×

4440

        code = terrno;

×

4441

        QUERY_CHECK_CODE(code, lino, _error);

×

4445

    void*   buff = NULL;

×

4446

    int32_t len = 0;

×

4448

        pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME,

×

4450

    if (res == TSDB_CODE_SUCCESS) {

×

4451

      code = doStreamSessionDecodeOpState(buff, len, pOperator, true, NULL);

×

4452

      taosMemoryFree(buff);

×

4453

      QUERY_CHECK_CODE(code, lino, _error);

×

4457

  if (!IS_FINAL_SESSION_OP(pOperator) || numOfChild == 0) {

×

4458

    pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;

×

4461

  *pOptrInfo = pOperator;

×

4462

  return code;

×

4464

_error:

×

4465

  if (pInfo != NULL) {

×

4466

    destroyStreamSessionAggOperatorInfo(pInfo);

×

4468

  if (pOperator != NULL) {

×

4469

    pOperator->info = NULL;

×

4470

    destroyOperator(pOperator);

×

4472

  pTaskInfo->code = code;

×

4473

  if (code != TSDB_CODE_SUCCESS) {

×

4474

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

4476

  return code;

×

4479

void destroyStreamStateOperatorInfo(void* param) {

×

4480

  if (param == NULL) {

×

4481

    return;

×

4483

  SStreamStateAggOperatorInfo* pInfo = (SStreamStateAggOperatorInfo*)param;

×

4484

  cleanupBasicInfo(&pInfo->binfo);

×

4485

  if (pInfo->pOperator) {

×

4486

    cleanupResultInfoInStream(pInfo->pOperator->pTaskInfo, pInfo->streamAggSup.pState, &pInfo->pOperator->exprSupp,

×

4488

    pInfo->pOperator = NULL;

×

4491

  destroyStreamBasicInfo(&pInfo->basic);

×

4492

  clearSessionGroupResInfo(&pInfo->groupResInfo);

×

4493

  taosArrayDestroyEx(pInfo->pUpdated, destroyResultWinInfo);

×

4494

  pInfo->pUpdated = NULL;

×

4495

  destroyStreamAggSupporter(&pInfo->streamAggSup);

×

4497

  cleanupExprSupp(&pInfo->scalarSupp);

×

4498

  if (pInfo->pChildren != NULL) {

×

4499

    int32_t size = taosArrayGetSize(pInfo->pChildren);

×

4500

    for (int32_t i = 0; i < size; i++) {

×

4501

      SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, i);

×

4502

      destroyOperator(pChild);

×

4504

    taosArrayDestroy(pInfo->pChildren);

×

4506

  colDataDestroy(&pInfo->twAggSup.timeWindowData);

×

4507

  blockDataDestroy(pInfo->pDelRes);

×

4508

  tSimpleHashCleanup(pInfo->pSeUpdated);

×

4509

  tSimpleHashCleanup(pInfo->pSeDeleted);

×

4510

  cleanupGroupResInfo(&pInfo->groupResInfo);

×

4512

  taosArrayDestroy(pInfo->historyWins);

×

4513

  blockDataDestroy(pInfo->pCheckpointRes);

×

4514

  tSimpleHashCleanup(pInfo->pPkDeleted);

×

4515

  destroyNonBlockAggSupptor(&pInfo->nbSup);

×

4517

  taosMemoryFreeClear(param);

×

4520

bool isTsInWindow(SStateWindowInfo* pWin, TSKEY ts) {

×

4521

  if (pWin->winInfo.sessionWin.win.skey <= ts && ts <= pWin->winInfo.sessionWin.win.ekey) {

×

4522

    return true;

×

4524

  return false;

×

4527

bool isEqualStateKey(SStateWindowInfo* pWin, char* pKeyData) {

×

4528

  return pKeyData && compareVal(pKeyData, pWin->pStateKey);

×

4531

bool compareStateKey(void* data, void* key) {

×

4532

  if (!data || !key) {

×

4533

    return true;

×

4535

  SStateKeys* stateKey = (SStateKeys*)key;

×

4536

  stateKey->pData = (char*)key + sizeof(SStateKeys);

×

4537

  return compareVal(data, stateKey);

×

4540

bool compareWinStateKey(SStateKeys* left, SStateKeys* right) {

×

4541

  if (!left || !right) {

×

4542

    return false;

×

4544

  return compareVal(left->pData, right);

×

4547

void getNextStateWin(const SStreamAggSupporter* pAggSup, SStateWindowInfo* pNextWin, bool asc) {

×

4548

  SStreamStateCur* pCur = NULL;

×

4550

  if (pAggSup == NULL || pNextWin == NULL) {

×

4551

    return;

×

4554

  if (asc)

×

4555

    pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pNextWin->winInfo.sessionWin);

×

4557

    pCur = pAggSup->stateStore.streamStateSessionSeekKeyPrev(pAggSup->pState, &pNextWin->winInfo.sessionWin);

×

4558

  int32_t nextSize = pAggSup->resultRowSize;

×

4559

  int32_t winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin,

×

4560

                                                                     (void**)&pNextWin->winInfo.pStatePos, &nextSize);

×

4561

  if (winCode != TSDB_CODE_SUCCESS) {

×

4562

    SET_SESSION_WIN_INVALID(pNextWin->winInfo);

×

4564

    pNextWin->pStateKey =

×

4565

        (SStateKeys*)((char*)pNextWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));

×

4566

    pNextWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);

×

4567

    pNextWin->pStateKey->type = pAggSup->stateKeyType;

×

4568

    pNextWin->pStateKey->pData = (char*)pNextWin->pStateKey + sizeof(SStateKeys);

×

4569

    pNextWin->pStateKey->isNull = false;

×

4570

    pNextWin->winInfo.isOutput = true;

×

4572

  pAggSup->stateStore.streamStateFreeCur(pCur);

×

4575

int32_t getStateWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SStateWindowInfo* pCurWin,

×

4577

  int32_t code = TSDB_CODE_SUCCESS;

×

4578

  int32_t lino = 0;

×

4579

  int32_t size = pAggSup->resultRowSize;

×

4580

  pCurWin->winInfo.sessionWin.groupId = pKey->groupId;

×

4581

  pCurWin->winInfo.sessionWin.win.skey = pKey->win.skey;

×

4582

  pCurWin->winInfo.sessionWin.win.ekey = pKey->win.ekey;

×

4583

  code = getSessionWindowInfoByKey(pAggSup, pKey, &pCurWin->winInfo);

×

4584

  QUERY_CHECK_CODE(code, lino, _end);

×

4585

  QUERY_CHECK_CONDITION((IS_VALID_SESSION_WIN(pCurWin->winInfo)), code, lino, _end,

×

4588

  pCurWin->pStateKey =

×

4589

      (SStateKeys*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));

×

4590

  pCurWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);

×

4591

  pCurWin->pStateKey->type = pAggSup->stateKeyType;

×

4592

  pCurWin->pStateKey->pData = (char*)pCurWin->pStateKey + sizeof(SStateKeys);

×

4593

  pCurWin->pStateKey->isNull = false;

×

4594

  pCurWin->winInfo.isOutput = true;

×

4595

  if (pCurWin->winInfo.pStatePos->needFree) {

×

4596

    pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->winInfo.sessionWin);

×

4599

  qDebug("===stream===get state cur win buff. skey:%" PRId64 ", endkey:%" PRId64, pCurWin->winInfo.sessionWin.win.skey,

×

4602

  pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin;

×

4603

  getNextStateWin(pAggSup, pNextWin, true);

×

4605

_end:

×

4606

  qDebug("===stream===get state next win buff. skey:%" PRId64 ", endkey:%" PRId64,

×

4608

  if (code != TSDB_CODE_SUCCESS) {

×

4609

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

4611

  return code;

×

4614

int32_t setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData,

×

4616

  int32_t size = pAggSup->resultRowSize;

×

4617

  pCurWin->winInfo.sessionWin.groupId = groupId;

×

4618

  pCurWin->winInfo.sessionWin.win.skey = ts;

×

4619

  pCurWin->winInfo.sessionWin.win.ekey = ts;

×

4620

  int32_t code = TSDB_CODE_SUCCESS;

×

4621

  int32_t lino = 0;

×

4622

  int32_t winCode = TSDB_CODE_SUCCESS;

×

4623

  code = pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, pKeyData,

×

4625

                                                           (void**)&pCurWin->winInfo.pStatePos, &size, &winCode);

×

4626

  QUERY_CHECK_CODE(code, lino, _end);

×

4628

  pCurWin->pStateKey =

×

4629

      (SStateKeys*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));

×

4630

  pCurWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);

×

4631

  pCurWin->pStateKey->type = pAggSup->stateKeyType;

×

4632

  pCurWin->pStateKey->pData = (char*)pCurWin->pStateKey + sizeof(SStateKeys);

×

4633

  pCurWin->pStateKey->isNull = false;

×

4635

  if (winCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->winInfo.sessionWin.win)) {

×

4636

    winCode = TSDB_CODE_FAILED;

×

4637

    clearOutputBuf(pAggSup->pState, pCurWin->winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);

×

4638

    pCurWin->pStateKey =

×

4639

        (SStateKeys*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));

×

4640

    pCurWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);

×

4641

    pCurWin->pStateKey->type = pAggSup->stateKeyType;

×

4642

    pCurWin->pStateKey->pData = (char*)pCurWin->pStateKey + sizeof(SStateKeys);

×

4643

    pCurWin->pStateKey->isNull = false;

×

4644

    pCurWin->winInfo.sessionWin.groupId = groupId;

×

4645

    pCurWin->winInfo.sessionWin.win.skey = ts;

×

4646

    pCurWin->winInfo.sessionWin.win.ekey = ts;

×

4647

    qDebug("===stream===reset state win key. skey:%" PRId64 ", endkey:%" PRId64, pCurWin->winInfo.sessionWin.win.skey,

×

4651

  if (winCode == TSDB_CODE_SUCCESS) {

×

4652

    pCurWin->winInfo.isOutput = true;

×

4653

    if (pCurWin->winInfo.pStatePos->needFree) {

×

4654

      pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->winInfo.sessionWin);

×

4656

  } else if (pKeyData) {

×

4657

    if (IS_VAR_DATA_TYPE(pAggSup->stateKeyType)) {

×

4658

      varDataCopy(pCurWin->pStateKey->pData, pKeyData);

×

4660

      memcpy(pCurWin->pStateKey->pData, pKeyData, pCurWin->pStateKey->bytes);

×

4664

  *pWinCode = winCode;

×

4666

  qDebug("===stream===set state cur win buff. skey:%" PRId64 ", endkey:%" PRId64, pCurWin->winInfo.sessionWin.win.skey,

×

4669

  pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin;

×

4670

  getNextStateWin(pAggSup, pNextWin, true);

×

4671

  qDebug("===stream===set state next win buff. skey:%" PRId64 ", endkey:%" PRId64,

×

4673

_end:

×

4674

  if (code != TSDB_CODE_SUCCESS) {

×

4675

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

4677

  return code;

×

4680

int32_t updateStateWindowInfo(SStreamAggSupporter* pAggSup, SStateWindowInfo* pWinInfo, SStateWindowInfo* pNextWin,

×

4684

  int32_t code = TSDB_CODE_SUCCESS;

×

4685

  int32_t lino = 0;

×

4686

  *allEqual = true;

×

4687

  for (int32_t i = start; i < rows; ++i) {

×

4688

    char* pKeyData = colDataGetData(pKeyCol, i);

×

4689

    if (!isTsInWindow(pWinInfo, pTs[i])) {

×

4690

      if (isEqualStateKey(pWinInfo, pKeyData)) {

×

4691

        if (IS_VALID_SESSION_WIN(pNextWin->winInfo)) {

×

4693

          if (pTs[i] >= pNextWin->winInfo.sessionWin.win.skey) {

×

4694

            (*pWinRows) = i - start;

×

4695

            goto _end;

×

4699

        (*pWinRows) = i - start;

×

4700

        goto _end;

×

4704

    if (pWinInfo->winInfo.sessionWin.win.skey > pTs[i]) {

×

4705

      if (pSeDeleted && pWinInfo->winInfo.isOutput) {

×

4706

        code = saveDeleteRes(pSeDeleted, pWinInfo->winInfo.sessionWin);

×

4707

        QUERY_CHECK_CODE(code, lino, _end);

×

4709

      removeSessionResult(pAggSup, pSeUpdated, pResultRows, &pWinInfo->winInfo.sessionWin);

×

4710

      pWinInfo->winInfo.sessionWin.win.skey = pTs[i];

×

4712

    pWinInfo->winInfo.sessionWin.win.ekey = TMAX(pWinInfo->winInfo.sessionWin.win.ekey, pTs[i]);

×

4713

    memcpy(pWinInfo->winInfo.pStatePos->pKey, &pWinInfo->winInfo.sessionWin, sizeof(SSessionKey));

×

4714

    if (!isEqualStateKey(pWinInfo, pKeyData)) {

×

4715

      *allEqual = false;

×

4718

  (*pWinRows) = rows - start;

×

4720

_end:

×

4721

  if (code != TSDB_CODE_SUCCESS) {

×

4722

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

4724

  return code;

×

4727

static bool isWinResult(SSessionKey* pKey, SSHashObj* pSeUpdate, SSHashObj* pResults) {

×

4728

  SSessionKey checkKey = {0};

×

4729

  getSessionHashKey(pKey, &checkKey);

×

4730

  if (tSimpleHashGet(pSeUpdate, &checkKey, sizeof(SSessionKey)) != NULL) {

×

4731

    return true;

×

4734

  if (tSimpleHashGet(pResults, &checkKey, sizeof(SSessionKey)) != NULL) {

×

4735

    return true;

×

4737

  return false;

×

4740

static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pSeUpdated,

×

4742

  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

×

4743

  SStorageAPI*   pAPI = &pOperator->pTaskInfo->storageAPI;

×

4745

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

4746

  int32_t                      numOfOutput = pOperator->exprSupp.numOfExprs;

×

4747

  uint64_t                     groupId = pSDataBlock->info.id.groupId;

×

4748

  int32_t                      code = TSDB_CODE_SUCCESS;

×

4749

  int32_t                      lino = 0;

×

4750

  TSKEY*                       tsCols = NULL;

×

4751

  SResultRow*                  pResult = NULL;

×

4752

  int32_t                      winRows = 0;

×

4753

  SStreamAggSupporter*         pAggSup = &pInfo->streamAggSup;

×

4754

  SStreamNotifyEventSupp*      pNotifySup = &pInfo->basic.notifyEventSup;

×

4755

  STaskNotifyEventStat*        pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;

×

4757

  pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);

×

4758

  pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;

×

4759

  if (pAggSup->winRange.ekey <= 0) {

×

4760

    pAggSup->winRange.ekey = INT64_MAX;

×

4763

  if (pSDataBlock->pDataBlock != NULL) {

×

4764

    SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);

×

4765

    if (!pColDataInfo) {

×

4766

      code = TSDB_CODE_FAILED;

×

4767

      QUERY_CHECK_CODE(code, lino, _end);

×

4769

    tsCols = (int64_t*)pColDataInfo->pData;

×

4771

    return;

×

4774

  int32_t rows = pSDataBlock->info.rows;

×

4775

  code = blockDataEnsureCapacity(pAggSup->pScanBlock, rows);

×

4776

  QUERY_CHECK_CODE(code, lino, _end);

×

4778

  SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);

×

4779

  for (int32_t i = 0; i < rows; i += winRows) {

×

4780

    if (pInfo->ignoreExpiredData && checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo,

×

4781

                                                     &pInfo->twAggSup, pSDataBlock->info.id.uid, tsCols[i], NULL, 0) ||

×

4782

        colDataIsNull_s(pKeyColInfo, i)) {

×

4783

      i++;

×

4784

      continue;

×

4786

    char*            pKeyData = colDataGetData(pKeyColInfo, i);

×

4787

    int32_t          winIndex = 0;

×

4788

    bool             allEqual = true;

×

4789

    SStateWindowInfo curWin = {0};

×

4790

    SStateWindowInfo nextWin = {0};

×

4791

    int32_t          winCode = TSDB_CODE_SUCCESS;

×

4792

    code = setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin, &winCode);

×

4793

    QUERY_CHECK_CODE(code, lino, _end);

×

4795

    if (winCode != TSDB_CODE_SUCCESS && pTaskInfo->streamInfo.eventTypes) {

×

4796

      SStateWindowInfo prevWin = {.winInfo.sessionWin = curWin.winInfo.sessionWin};

×

4797

      getNextStateWin(pAggSup, &prevWin, false);

×

4798

      qDebug("===stream===get state prev win buff. skey:%" PRId64 ", endkey:%" PRId64,

×

4800

      releaseOutputBuf(pAggSup->pState, prevWin.winInfo.pStatePos, &pAPI->stateStore);

×

4803

      if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE) &&

×

4804

          IS_VALID_SESSION_WIN(prevWin.winInfo)) {

×

4805

        code = addStateAggNotifyEvent(SNOTIFY_EVENT_WINDOW_CLOSE, &prevWin.winInfo.sessionWin, prevWin.pStateKey,

×

4806

                                      curWin.pStateKey, true, pNotifySup, pNotifyEventStat);

×

4807

        QUERY_CHECK_CODE(code, lino, _end);

×

4809

      if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {

×

4810

        code = addStateAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &curWin.winInfo.sessionWin, curWin.pStateKey,

×

4811

                                      prevWin.pStateKey, false, pNotifySup, pNotifyEventStat);

×

4812

        QUERY_CHECK_CODE(code, lino, _end);

×

4816

    if (isWinResult(&nextWin.winInfo.sessionWin, pSeUpdated, pAggSup->pResultRows) == false) {

×

4817

      releaseOutputBuf(pAggSup->pState, nextWin.winInfo.pStatePos, &pAPI->stateStore);

×

4820

    setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo);

×

4821

    code = updateStateWindowInfo(pAggSup, &curWin, &nextWin, tsCols, groupId, pKeyColInfo, rows, i, &allEqual,

×

4823

    QUERY_CHECK_CODE(code, lino, _end);

×

4825

    if (!allEqual) {

×

4826

      uint64_t uid = 0;

×

4827

      code = appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey,

×

4829

      QUERY_CHECK_CODE(code, lino, _end);

×

4830

      int32_t tmpRes = tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey));

×

4831

      qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

4833

      doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin);

×

4834

      releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore);

×

4835

      continue;

×

4838

    code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput,

×

4840

    QUERY_CHECK_CODE(code, lino, _end);

×

4842

    code = saveSessionOutputBuf(pAggSup, &curWin.winInfo);

×

4843

    QUERY_CHECK_CODE(code, lino, _end);

×

4845

    if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_STATE_OP(pOperator)) {

×

4846

      code = saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin);

×

4847

      QUERY_CHECK_CODE(code, lino, _end);

×

4850

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {

×

4851

      code = saveResult(curWin.winInfo, pSeUpdated);

×

4852

      QUERY_CHECK_CODE(code, lino, _end);

×

4855

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

4856

      curWin.winInfo.pStatePos->beUpdated = true;

×

4857

      SSessionKey key = {0};

×

4858

      getSessionHashKey(&curWin.winInfo.sessionWin, &key);

×

4860

          tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo));

×

4861

      QUERY_CHECK_CODE(code, lino, _end);

×

4866

    if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE)) {

×

4867

      code = addStateAggNotifyEvent(SNOTIFY_EVENT_WINDOW_CLOSE, &curWin.winInfo.sessionWin, curWin.pStateKey,

×

4868

                                    nextWin.pStateKey, false, pNotifySup, pNotifyEventStat);

×

4869

      QUERY_CHECK_CODE(code, lino, _end);

×

4873

_end:

×

4874

  if (code != TSDB_CODE_SUCCESS) {

×

4875

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

4879

int32_t doStreamStateEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOperator, bool isParent) {

×

4880

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

4881

  if (!pInfo) {

×

4882

    return 0;

×

4885

  void* pData = (buf == NULL) ? NULL : *buf;

×

4888

  int32_t tlen = 0;

×

4889

  int32_t mapSize = tSimpleHashGetSize(pInfo->streamAggSup.pResultRows);

×

4890

  tlen += taosEncodeFixedI32(buf, mapSize);

×

4891

  void*   pIte = NULL;

×

4892

  size_t  keyLen = 0;

×

4893

  int32_t iter = 0;

×

4894

  while ((pIte = tSimpleHashIterate(pInfo->streamAggSup.pResultRows, pIte, &iter)) != NULL) {

×

4895

    void* key = tSimpleHashGetKey(pIte, &keyLen);

×

4896

    tlen += encodeSSessionKey(buf, key);

×

4897

    tlen += encodeSResultWindowInfo(buf, pIte, pInfo->streamAggSup.resultRowSize);

×

4901

  tlen += encodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);

×

4904

  int32_t size = taosArrayGetSize(pInfo->pChildren);

×

4905

  tlen += taosEncodeFixedI32(buf, size);

×

4906

  for (int32_t i = 0; i < size; i++) {

×

4907

    SOperatorInfo* pChOp = taosArrayGetP(pInfo->pChildren, i);

×

4908

    tlen += doStreamStateEncodeOpState(buf, 0, pChOp, false);

×

4912

  tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);

×

4915

  tlen += encodeStreamBasicInfo(buf, &pInfo->basic);

×

4918

  if (isParent) {

×

4919

    if (buf) {

×

4920

      uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));

×

4921

      tlen += taosEncodeFixedU32(buf, cksum);

×

4923

      tlen += sizeof(uint32_t);

×

4927

  return tlen;

×

4930

int32_t doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator, bool isParent, void** ppBuf) {

×

4931

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

4932

  int32_t                      code = TSDB_CODE_SUCCESS;

×

4933

  int32_t                      lino = 0;

×

4934

  SStreamAggSupporter*         pAggSup = &pInfo->streamAggSup;

×

4935

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

4936

  void*                        pDataEnd = POINTER_SHIFT(buf, len);

×

4937

  if (!pInfo) {

×

4938

    code = TSDB_CODE_FAILED;

×

4939

    QUERY_CHECK_CODE(code, lino, _end);

×

4943

  if (isParent) {

×

4944

    int32_t dataLen = len - sizeof(uint32_t);

×

4945

    void*   pCksum = POINTER_SHIFT(buf, dataLen);

×

4946

    if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) {

×

4947

      qError("stream state_window state is invalid");

×

4948

      code = TSDB_CODE_FAILED;

×

4949

      QUERY_CHECK_CODE(code, lino, _end);

×

4951

    pDataEnd = pCksum;

×

4955

  int32_t mapSize = 0;

×

4956

  buf = taosDecodeFixedI32(buf, &mapSize);

×

4957

  for (int32_t i = 0; i < mapSize; i++) {

×

4958

    SResultWindowInfo winfo = {0};

×

4959

    buf = decodeSSessionKey(buf, &winfo.sessionWin);

×

4960

    int32_t winCode = TSDB_CODE_SUCCESS;

×

4961

    code = pAggSup->stateStore.streamStateStateAddIfNotExist(

×

4964

    QUERY_CHECK_CODE(code, lino, _end);

×

4966

    buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize);

×

4967

    code = tSimpleHashPut(pInfo->streamAggSup.pResultRows, &winfo.sessionWin, sizeof(SSessionKey), &winfo,

×

4969

    QUERY_CHECK_CODE(code, lino, _end);

×

4973

  buf = decodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);

×

4976

  int32_t size = 0;

×

4977

  buf = taosDecodeFixedI32(buf, &size);

×

4978

  for (int32_t i = 0; i < size; i++) {

×

4979

    SOperatorInfo* pChOp = taosArrayGetP(pInfo->pChildren, i);

×

4980

    code = doStreamStateDecodeOpState(buf, 0, pChOp, false, &buf);

×

4981

    QUERY_CHECK_CODE(code, lino, _end);

×

4985

  buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);

×

4987

  if (ppBuf) {

×

4988

    (*ppBuf) = buf;

×

4992

  if (buf < pDataEnd) {

×

4993

    code = decodeStreamBasicInfo(&buf, &pInfo->basic);

×

4994

    QUERY_CHECK_CODE(code, lino, _end);

×

4997

_end:

×

4998

  if (code != TSDB_CODE_SUCCESS) {

×

4999

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

5001

  return code;

×

5004

void doStreamStateSaveCheckpoint(SOperatorInfo* pOperator) {

×

5005

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

5006

  if (needSaveStreamOperatorInfo(&pInfo->basic)) {

×

5007

    int32_t len = doStreamStateEncodeOpState(NULL, 0, pOperator, true);

×

5008

    void*   buf = taosMemoryCalloc(1, len);

×

5009

    if (!buf) {

×

5010

      qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY));

×

5011

      return;

×

5013

    void* pBuf = buf;

×

5014

    len = doStreamStateEncodeOpState(&pBuf, len, pOperator, true);

×

5015

    pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_CHECKPOINT_NAME,

×

5017

    taosMemoryFree(buf);

×

5018

    saveStreamOperatorStateComplete(&pInfo->basic);

×

5022

static int32_t buildStateResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

5023

  int32_t                      code = TSDB_CODE_SUCCESS;

×

5024

  int32_t                      lino = 0;

×

5025

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

5026

  SOptrBasicInfo*              pBInfo = &pInfo->binfo;

×

5027

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

5028

  SStreamNotifyEventSupp*      pNotifySup = &pInfo->basic.notifyEventSup;

×

5029

  STaskNotifyEventStat*        pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;

×

5030

  bool                         addNotifyEvent = false;

×

5031

  addNotifyEvent = BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);

×

5032

  doBuildDeleteDataBlock(pOperator, pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator, &pInfo->groupResInfo);

×

5033

  if (pInfo->pDelRes->info.rows > 0) {

×

5034

    printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5035

    if (addNotifyEvent) {

×

5036

      code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);

×

5037

      QUERY_CHECK_CODE(code, lino, _end);

×

5039

    (*ppRes) = pInfo->pDelRes;

×

5040

    return code;

×

5043

  doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes,

×

5045

  if (pBInfo->pRes->info.rows > 0) {

×

5046

    printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5047

    if (addNotifyEvent) {

×

5048

      code = addAggResultNotifyEvent(pBInfo->pRes, pNotifySup->pSessionKeys, pTaskInfo->streamInfo.notifyResultSchema,

×

5050

      QUERY_CHECK_CODE(code, lino, _end);

×

5052

    (*ppRes) = pBInfo->pRes;

×

5053

    return code;

×

5056

  code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);

×

5057

  QUERY_CHECK_CODE(code, lino, _end);

×

5058

  if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {

×

5059

    printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5060

    (*ppRes) = pNotifySup->pEventBlock;

×

5061

    return code;

×

5064

  code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);

×

5065

  QUERY_CHECK_CODE(code, lino, _end);

×

5067

_end:

×

5068

  if (code != TSDB_CODE_SUCCESS) {

×

5069

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

5071

  (*ppRes) = NULL;

×

5072

  return code;

×

5075

static int32_t doStreamStateAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

5076

  if (pOperator->status == OP_EXEC_DONE) {

×

5077

    (*ppRes) = NULL;

×

5078

    return TSDB_CODE_SUCCESS;

×

5081

  int32_t                      code = TSDB_CODE_SUCCESS;

×

5082

  int32_t                      lino = 0;

×

5083

  SExprSupp*                   pSup = &pOperator->exprSupp;

×

5084

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

5085

  SOptrBasicInfo*              pBInfo = &pInfo->binfo;

×

5086

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

5087

  qDebug("===stream=== stream state agg");

×

5088

  if (pOperator->status == OP_RES_TO_RETURN) {

×

5089

    SSDataBlock* resBlock = NULL;

×

5090

    code = buildStateResult(pOperator, &resBlock);

×

5091

    QUERY_CHECK_CODE(code, lino, _end);

×

5092

    if (resBlock != NULL) {

×

5093

      (*ppRes) = resBlock;

×

5094

      return code;

×

5097

    if (pInfo->recvGetAll) {

×

5098

      pInfo->recvGetAll = false;

×

5099

      resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows);

×

5102

    if (pInfo->reCkBlock) {

×

5103

      pInfo->reCkBlock = false;

×

5104

      printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5105

      (*ppRes) = pInfo->pCheckpointRes;

×

5106

      return code;

×

5109

    setStreamOperatorCompleted(pOperator);

×

5110

    (*ppRes) = NULL;

×

5111

    return code;

×

5114

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

5115

  if (!pInfo->pUpdated) {

×

5116

    pInfo->pUpdated = taosArrayInit(16, sizeof(SResultWindowInfo));

×

5117

    QUERY_CHECK_NULL(pInfo->pUpdated, code, lino, _end, terrno);

×

5119

  if (!pInfo->pSeUpdated) {

×

5120

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

5121

    pInfo->pSeUpdated = tSimpleHashInit(64, hashFn);

×

5122

    QUERY_CHECK_NULL(pInfo->pSeUpdated, code, lino, _end, terrno);

×

5124

  while (1) {

×

5125

    SSDataBlock* pBlock = NULL;

×

5126

    code = downstream->fpSet.getNextFn(downstream, &pBlock);

×

5127

    QUERY_CHECK_CODE(code, lino, _end);

×

5129

    if (pBlock == NULL) {

×

5130

      break;

×

5132

    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));

×

5133

    setStreamOperatorState(&pInfo->basic, pBlock->info.type);

×

5135

    if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||

×

5136

        pBlock->info.type == STREAM_CLEAR) {

×

5137

      bool add = pInfo->destHasPrimaryKey && IS_NORMAL_STATE_OP(pOperator);

×

5138

      code = deleteSessionWinState(&pInfo->streamAggSup, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted,

×

5140

      QUERY_CHECK_CODE(code, lino, _end);

×

5141

      continue;

×

5142

    } else if (pBlock->info.type == STREAM_GET_ALL) {

×

5143

      pInfo->recvGetAll = true;

×

5144

      code = getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pSeUpdated);

×

5145

      QUERY_CHECK_CODE(code, lino, _end);

×

5146

      continue;

×

5147

    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {

×

5148

      (*ppRes) = pBlock;

×

5149

      return code;

×

5150

    } else if (pBlock->info.type == STREAM_CHECKPOINT) {

×

5151

      pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState);

×

5152

      doStreamStateSaveCheckpoint(pOperator);

×

5153

      code = copyDataBlock(pInfo->pCheckpointRes, pBlock);

×

5154

      QUERY_CHECK_CODE(code, lino, _end);

×

5156

      continue;

×

5158

      if (pBlock->info.type != STREAM_NORMAL && pBlock->info.type != STREAM_INVALID) {

×

5159

        code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;

×

5160

        QUERY_CHECK_CODE(code, lino, _end);

×

5164

    if (pInfo->scalarSupp.pExprInfo != NULL) {

×

5165

      SExprSupp* pExprSup = &pInfo->scalarSupp;

×

5166

      code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);

×

5167

      QUERY_CHECK_CODE(code, lino, _end);

×

5170

    code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

5171

    QUERY_CHECK_CODE(code, lino, _end);

×

5172

    doStreamStateAggImpl(pOperator, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted);

×

5173

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);

×

5176

  pOperator->status = OP_RES_TO_RETURN;

×

5178

  code = closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pInfo->pSeUpdated);

×

5179

  QUERY_CHECK_CODE(code, lino, _end);

×

5181

  code = copyUpdateResult(&pInfo->pSeUpdated, pInfo->pUpdated, sessionKeyCompareAsc);

×

5182

  QUERY_CHECK_CODE(code, lino, _end);

×

5184

  removeSessionDeleteResults(pInfo->pSeDeleted, pInfo->pUpdated);

×

5186

  if (pInfo->isHistoryOp) {

×

5187

    code = getMaxTsWins(pInfo->pUpdated, pInfo->historyWins);

×

5188

    QUERY_CHECK_CODE(code, lino, _end);

×

5190

  if (pInfo->destHasPrimaryKey && IS_NORMAL_STATE_OP(pOperator)) {

×

5191

    code = copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pSeDeleted);

×

5192

    QUERY_CHECK_CODE(code, lino, _end);

×

5195

  initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);

×

5196

  pInfo->pUpdated = NULL;

×

5197

  code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);

×

5198

  QUERY_CHECK_CODE(code, lino, _end);

×

5200

  SSDataBlock* resBlock = NULL;

×

5201

  code = buildStateResult(pOperator, &resBlock);

×

5202

  QUERY_CHECK_CODE(code, lino, _end);

×

5203

  if (resBlock != NULL) {

×

5204

    (*ppRes) = resBlock;

×

5205

    return code;

×

5208

_end:

×

5209

  if (code != TSDB_CODE_SUCCESS) {

×

5210

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

5211

    pTaskInfo->code = code;

×

5212

    T_LONG_JMP(pTaskInfo->env, code);

×

5214

  setStreamOperatorCompleted(pOperator);

×

5215

  (*ppRes) = NULL;

×

5216

  return code;

×

5219

static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {

×

5220

  SSDataBlock* pRes = NULL;

×

5221

  int32_t      code = doStreamStateAggNext(pOperator, &pRes);

×

5222

  return pRes;

×

5225

void streamStateReleaseState(SOperatorInfo* pOperator) {

×

5226

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

5227

  int32_t                      winSize = taosArrayGetSize(pInfo->historyWins) * sizeof(SSessionKey);

×

5228

  int32_t                      resSize = winSize + sizeof(TSKEY);

×

5229

  char*                        pBuff = taosMemoryCalloc(1, resSize);

×

5230

  if (!pBuff) {

×

5231

    return;

×

5233

  memcpy(pBuff, pInfo->historyWins->pData, winSize);

×

5234

  memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY));

×

5235

  qDebug("===stream=== relase state. save result count:%d", (int32_t)taosArrayGetSize(pInfo->historyWins));

×

5236

  pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_STATE_NAME,

×

5238

  pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState);

×

5239

  taosMemoryFreeClear(pBuff);

×

5241

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

5242

  if (downstream->fpSet.releaseStreamStateFn) {

×

5243

    downstream->fpSet.releaseStreamStateFn(downstream);

×

5247

int32_t compactStateWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin,

×

5249

  SExprSupp*                   pSup = &pOperator->exprSupp;

×

5250

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

5251

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

5252

  return compactTimeWindow(pSup, &pInfo->streamAggSup, &pInfo->twAggSup, pTaskInfo, pCurWin, pNextWin, pStUpdated,

×

5256

void streamStateReloadState(SOperatorInfo* pOperator) {

×

5257

  int32_t                      code = TSDB_CODE_SUCCESS;

×

5258

  int32_t                      lino = 0;

×

5259

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

5260

  SStreamAggSupporter*         pAggSup = &pInfo->streamAggSup;

×

5261

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

5262

  resetWinRange(&pAggSup->winRange);

×

5264

  SSessionKey seKey = {.win.skey = INT64_MIN, .win.ekey = INT64_MIN, .groupId = 0};

×

5265

  int32_t     size = 0;

×

5266

  void*       pBuf = NULL;

×

5267

  code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_STATE_OP_STATE_NAME,

×

5269

  QUERY_CHECK_CODE(code, lino, _end);

×

5271

  int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);

×

5272

  qDebug("===stream=== reload state. get result count:%d", num);

×

5273

  SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;

×

5275

  TSKEY ts = *(TSKEY*)((char*)pBuf + size - sizeof(TSKEY));

×

5276

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);

×

5277

  pAggSup->stateStore.streamStateReloadInfo(pAggSup->pState, ts);

×

5279

  if (!pInfo->pSeUpdated && num > 0) {

×

5280

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

5281

    pInfo->pSeUpdated = tSimpleHashInit(64, hashFn);

×

5282

    QUERY_CHECK_NULL(pInfo->pSeUpdated, code, lino, _end, terrno);

×

5284

  if (!pInfo->pSeDeleted && num > 0) {

×

5285

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

5286

    pInfo->pSeDeleted = tSimpleHashInit(64, hashFn);

×

5287

    QUERY_CHECK_NULL(pInfo->pSeDeleted, code, lino, _end, terrno);

×

5289

  for (int32_t i = 0; i < num; i++) {

×

5290

    SStateWindowInfo curInfo = {0};

×

5291

    SStateWindowInfo nextInfo = {0};

×

5292

    qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey,

×

5294

    code = getStateWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo, &nextInfo);

×

5295

    QUERY_CHECK_CODE(code, lino, _end);

×

5297

    bool cpRes = compareWinStateKey(curInfo.pStateKey, nextInfo.pStateKey);

×

5298

    qDebug("===stream=== reload state. next window info %" PRId64 ", %" PRIu64 ", compare:%d",

×

5300

    if (cpRes) {

×

5301

      code = compactStateWindow(pOperator, &curInfo.winInfo, &nextInfo.winInfo, pInfo->pSeUpdated, pInfo->pSeDeleted);

×

5302

      qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey,

×

5304

      QUERY_CHECK_CODE(code, lino, _end);

×

5306

      if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {

×

5307

        code = saveResult(curInfo.winInfo, pInfo->pSeUpdated);

×

5308

        QUERY_CHECK_CODE(code, lino, _end);

×

5309

      } else if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

5310

        if (!isCloseWindow(&curInfo.winInfo.sessionWin.win, &pInfo->twAggSup)) {

×

5311

          code = saveDeleteRes(pInfo->pSeDeleted, curInfo.winInfo.sessionWin);

×

5312

          QUERY_CHECK_CODE(code, lino, _end);

×

5314

        SSessionKey key = {0};

×

5315

        getSessionHashKey(&curInfo.winInfo.sessionWin, &key);

×

5316

        code = tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curInfo.winInfo,

×

5318

        QUERY_CHECK_CODE(code, lino, _end);

×

5320

    } else if (IS_VALID_SESSION_WIN(nextInfo.winInfo)) {

×

5321

      releaseOutputBuf(pAggSup->pState, nextInfo.winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);

×

5324

    if (IS_VALID_SESSION_WIN(curInfo.winInfo)) {

×

5325

      code = saveSessionOutputBuf(pAggSup, &curInfo.winInfo);

×

5326

      QUERY_CHECK_CODE(code, lino, _end);

×

5329

  taosMemoryFreeClear(pBuf);

×

5331

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

5332

  if (downstream->fpSet.reloadStreamStateFn) {

×

5333

    downstream->fpSet.reloadStreamStateFn(downstream);

×

5335

  reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup);

×

5337

_end:

×

5338

  taosMemoryFreeClear(pBuf);

×

5339

  if (code != TSDB_CODE_SUCCESS) {

×

5340

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

5342

×

5344

int32_t createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,

×

5346

  QRY_PARAM_CHECK(pOptrInfo);

×

5347

  int32_t code = 0;

×

5348

  int32_t lino = 0;

×

5350

  SStreamStateWinodwPhysiNode* pStateNode = (SStreamStateWinodwPhysiNode*)pPhyNode;

×

5351

  int32_t                      tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;

×

5352

  SColumnNode*                 pColNode = (SColumnNode*)(pStateNode->pStateKey);

×

5353

  SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo));

×

5354

  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));

×

5355

  if (pInfo == NULL || pOperator == NULL) {

×

5356

    code = terrno;

×

5357

    QUERY_CHECK_CODE(code, lino, _error);

×

5360

  pInfo->stateCol = extractColumnFromColumnNode(pColNode);

×

5361

  initResultSizeInfo(&pOperator->resultInfo, 4096);

×

5362

  if (pStateNode->window.pExprs != NULL) {

×

5363

    int32_t    numOfScalar = 0;

×

5364

    SExprInfo* pScalarExprInfo = NULL;

×

5365

    code = createExprInfo(pStateNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalar);

×

5366

    QUERY_CHECK_CODE(code, lino, _error);

×

5368

    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);

×

5369

    QUERY_CHECK_CODE(code, lino, _error);

×

5372

  pInfo->twAggSup = (STimeWindowAggSupp){

×

5373

      .waterMark = pStateNode->window.watermark,

×

5374

      .calTrigger = pStateNode->window.triggerType,

×

5377

      .deleteMark = getDeleteMark(&pStateNode->window, 0),

×

5380

  code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

×

5381

  QUERY_CHECK_CODE(code, lino, _error);

×

5383

  SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);

×

5384

  QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);

×

5385

  pInfo->binfo.pRes = pResBlock;

×

5387

  SExprSupp* pExpSup = &pOperator->exprSupp;

×

5388

  int32_t    numOfCols = 0;

×

5389

  SExprInfo* pExprInfo = NULL;

×

5390

  code = createExprInfo(pStateNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);

×

5391

  QUERY_CHECK_CODE(code, lino, _error);

×

5393

  code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore);

×

5394

  if (code != TSDB_CODE_SUCCESS) {

×

5395

    goto _error;

×

5397

  int32_t keySize = sizeof(SStateKeys) + pColNode->node.resType.bytes;

×

5398

  int16_t type = pColNode->node.resType.type;

×

5399

  pInfo->primaryTsIndex = tsSlotId;

×

5401

      initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState, keySize, type,

×

5402

                             &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup, GET_TASKID(pTaskInfo),

×

5404

  QUERY_CHECK_CODE(code, lino, _error);

×

5406

  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

5407

  pInfo->pSeDeleted = tSimpleHashInit(64, hashFn);

×

5408

  QUERY_CHECK_NULL(pInfo->pSeDeleted, code, lino, _error, terrno);

×

5409

  pInfo->pDelIterator = NULL;

×

5411

  code = createSpecialDataBlock(STREAM_DELETE_RESULT, &pInfo->pDelRes);

×

5412

  QUERY_CHECK_CODE(code, lino, _error);

×

5414

  pInfo->pChildren = NULL;

×

5415

  pInfo->ignoreExpiredData = pStateNode->window.igExpired;

×

5416

  pInfo->ignoreExpiredDataSaved = false;

×

5417

  pInfo->pUpdated = NULL;

×

5418

  pInfo->pSeUpdated = NULL;

×

5419

  pInfo->dataVersion = 0;

×

5420

  pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey));

×

5421

  if (!pInfo->historyWins) {

×

5422

    code = terrno;

×

5423

    QUERY_CHECK_CODE(code, lino, _error);

×

5426

  if (pHandle) {

×

5427

    pInfo->isHistoryOp = (pHandle->fillHistory == STREAM_HISTORY_OPERATOR);

×

5430

  code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);

×

5431

  QUERY_CHECK_CODE(code, lino, _error);

×

5433

  pInfo->recvGetAll = false;

×

5434

  pInfo->pPkDeleted = tSimpleHashInit(64, hashFn);

×

5435

  QUERY_CHECK_NULL(pInfo->pPkDeleted, code, lino, _error, terrno);

×

5436

  pInfo->destHasPrimaryKey = pStateNode->window.destHasPrimaryKey;

×

5437

  pInfo->pOperator = pOperator;

×

5439

  setOperatorInfo(pOperator, "StreamStateAggOperator", nodeType(pPhyNode), true, OP_NOT_OPENED,

×

5442

  void*   buff = NULL;

×

5443

  int32_t len = 0;

×

5445

      pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_CHECKPOINT_NAME,

×

5447

  if (res == TSDB_CODE_SUCCESS) {

×

5448

    code = doStreamStateDecodeOpState(buff, len, pOperator, true, NULL);

×

5449

    taosMemoryFree(buff);

×

5450

    QUERY_CHECK_CODE(code, lino, _error);

×

5452

  initNonBlockAggSupptor(&pInfo->nbSup, NULL, NULL);

×

5453

  code = initStreamBasicInfo(&pInfo->basic, pOperator);

×

5454

  QUERY_CHECK_CODE(code, lino, _error);

×

5456

  if (pStateNode->window.triggerType == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {

×

5457

    if (pHandle->fillHistory == STREAM_HISTORY_OPERATOR) {

×

5458

      setFillHistoryOperatorFlag(&pInfo->basic);

×

5459

    } else if (pHandle->fillHistory == STREAM_RECALCUL_OPERATOR) {

×

5460

      setRecalculateOperatorFlag(&pInfo->basic);

×

5464

        createOperatorFpSet(optrDummyOpenFn, doStreamStateNonblockAggNext, NULL, destroyStreamStateOperatorInfo,

×

5466

    setOperatorStreamStateFn(pOperator, streamStateNonblockReleaseState, streamStateNonblockReloadState);

×

5468

    pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAggNext, NULL, destroyStreamStateOperatorInfo,

×

5470

    setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState);

×

5473

  code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,

×

5474

                        &pInfo->twAggSup, &pInfo->basic, 0);

×

5475

  QUERY_CHECK_CODE(code, lino, _error);

×

5477

  code = appendDownstream(pOperator, &downstream, 1);

×

5478

  QUERY_CHECK_CODE(code, lino, _error);

×

5480

  pInfo->trueForLimit = pStateNode->trueForLimit;

×

5482

  *pOptrInfo = pOperator;

×

5483

  return TSDB_CODE_SUCCESS;

×

5485

_error:

×

5486

  if (pInfo != NULL) destroyStreamStateOperatorInfo(pInfo);

×

5487

  destroyOperatorAndDownstreams(pOperator, &downstream, 1);

×

5488

  pTaskInfo->code = code;

×

5489

  qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

5490

  return code;

×

5505

static int32_t doStreamIntervalAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

5506

  int32_t                      code = TSDB_CODE_SUCCESS;

×

5507

  int32_t                      lino = 0;

×

5508

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

5509

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

5510

  SStorageAPI*                 pAPI = &pOperator->pTaskInfo->storageAPI;

×

5511

  SExprSupp*                   pSup = &pOperator->exprSupp;

×

5513

  qDebug("stask:%s  %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);

×

5515

  if (pOperator->status == OP_EXEC_DONE) {

×

5516

    (*ppRes) = NULL;

×

5517

    return code;

×

5520

  if (pOperator->status == OP_RES_TO_RETURN) {

×

5521

    SSDataBlock* resBlock = NULL;

×

5522

    code = buildIntervalResult(pOperator, &resBlock);

×

5523

    QUERY_CHECK_CODE(code, lino, _end);

×

5524

    if (resBlock != NULL) {

×

5525

      (*ppRes) = resBlock;

×

5526

      return code;

×

5529

    if (pInfo->recvGetAll) {

×

5530

      pInfo->recvGetAll = false;

×

5531

      resetUnCloseWinInfo(pInfo->aggSup.pResultRowHashTable);

×

5534

    if (pInfo->reCkBlock) {

×

5535

      pInfo->reCkBlock = false;

×

5536

      printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5537

      (*ppRes) = pInfo->pCheckpointRes;

×

5538

      return code;

×

5541

    setStreamOperatorCompleted(pOperator);

×

5542

    (*ppRes) = NULL;

×

5543

    return code;

×

5546

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

5548

  if (!pInfo->pUpdated) {

×

5549

    pInfo->pUpdated = taosArrayInit(4096, POINTER_BYTES);

×

5550

    QUERY_CHECK_NULL(pInfo->pUpdated, code, lino, _end, terrno);

×

5553

  if (!pInfo->pUpdatedMap) {

×

5554

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

5555

    pInfo->pUpdatedMap = tSimpleHashInit(4096, hashFn);

×

5556

    QUERY_CHECK_NULL(pInfo->pUpdatedMap, code, lino, _end, terrno);

×

5559

  while (1) {

×

5560

    SSDataBlock* pBlock = NULL;

×

5561

    code = downstream->fpSet.getNextFn(downstream, &pBlock);

×

5562

    QUERY_CHECK_CODE(code, lino, _end);

×

5564

    if (pBlock == NULL) {

×

5565

      qDebug("===stream===return data:%s. recv datablock num:%" PRIu64, getStreamOpName(pOperator->operatorType),

×

5567

      pInfo->numOfDatapack = 0;

×

5568

      break;

×

5571

    pInfo->numOfDatapack++;

×

5572

    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));

×

5573

    setStreamOperatorState(&pInfo->basic, pBlock->info.type);

×

5575

    if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||

×

5576

        pBlock->info.type == STREAM_CLEAR) {

×

5577

      code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, pInfo->pDelWins, pInfo->pUpdatedMap, NULL);

×

5578

      QUERY_CHECK_CODE(code, lino, _end);

×

5579

      continue;

×

5580

    } else if (pBlock->info.type == STREAM_GET_ALL) {

×

5581

      pInfo->recvGetAll = true;

×

5582

      code = getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap);

×

5583

      QUERY_CHECK_CODE(code, lino, _end);

×

5584

      continue;

×

5585

    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {

×

5586

      printDataBlock(pBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5587

      (*ppRes) = pBlock;

×

5588

      return code;

×

5589

    } else if (pBlock->info.type == STREAM_DROP_CHILD_TABLE) {

×

5590

      doDeleteWindowByGroupId(pOperator, pBlock);

×

5591

      printDataBlock(pBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5592

      (*ppRes) = pBlock;

×

5593

      return code;

×

5594

    } else if (pBlock->info.type == STREAM_CHECKPOINT) {

×

5595

      pAPI->stateStore.streamStateCommit(pInfo->pState);

×

5596

      doStreamIntervalSaveCheckpoint(pOperator);

×

5597

      pInfo->reCkBlock = true;

×

5598

      code = copyDataBlock(pInfo->pCheckpointRes, pBlock);

×

5599

      QUERY_CHECK_CODE(code, lino, _end);

×

5601

      continue;

×

5603

      if (pBlock->info.type != STREAM_NORMAL && pBlock->info.type != STREAM_INVALID) {

×

5604

        qDebug("===stream===%s ignore recv block. type:%d", GET_TASKID(pTaskInfo), pBlock->info.type);

×

5605

        continue;

×

5609

    if (pBlock->info.type == STREAM_NORMAL && pBlock->info.version != 0) {

×

5611

      pTaskInfo->version = pBlock->info.version;

×

5614

    if (pInfo->scalarSupp.pExprInfo != NULL) {

×

5615

      SExprSupp* pExprSup = &pInfo->scalarSupp;

×

5616

      code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);

×

5617

      QUERY_CHECK_CODE(code, lino, _end);

×

5623

    code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

5624

    QUERY_CHECK_CODE(code, lino, _end);

×

5631

    code = doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pInfo->pUpdatedMap, pInfo->pDeletedMap);

×

5632

    if (code == TSDB_CODE_STREAM_INTERNAL_ERROR) {

×

5633

      pOperator->status = OP_RES_TO_RETURN;

×

5634

      code = TSDB_CODE_SUCCESS;

×

5635

      break;

×

5637

    QUERY_CHECK_CODE(code, lino, _end);

×

5638

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);

×

5639

    pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey);

×

5641

  pOperator->status = OP_RES_TO_RETURN;

×

5642

  if (!pInfo->destHasPrimaryKey) {

×

5643

    removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins);

×

5645

  code = closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL,

×

5647

  QUERY_CHECK_CODE(code, lino, _end);

×

5649

  if (pInfo->destHasPrimaryKey && IS_NORMAL_INTERVAL_OP(pOperator)) {

×

5650

    code = copyIntervalDeleteKey(pInfo->pDeletedMap, pInfo->pDelWins);

×

5651

    QUERY_CHECK_CODE(code, lino, _end);

×

5654

  void*   pIte = NULL;

×

5655

  int32_t iter = 0;

×

5656

  while ((pIte = tSimpleHashIterate(pInfo->pUpdatedMap, pIte, &iter)) != NULL) {

×

5657

    void* tmp = taosArrayPush(pInfo->pUpdated, pIte);

×

5658

    if (!tmp) {

×

5659

      code = terrno;

×

5660

      QUERY_CHECK_CODE(code, lino, _end);

×

5663

  taosArraySort(pInfo->pUpdated, winPosCmprImpl);

×

5665

  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);

×

5666

  pInfo->pUpdated = NULL;

×

5667

  code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);

×

5668

  QUERY_CHECK_CODE(code, lino, _end);

×

5670

  tSimpleHashCleanup(pInfo->pUpdatedMap);

×

5671

  pInfo->pUpdatedMap = NULL;

×

5673

  code = buildIntervalResult(pOperator, ppRes);

×

5674

  QUERY_CHECK_CODE(code, lino, _end);

×

5676

  return code;

×

5678

_end:

×

5679

  if (code != TSDB_CODE_SUCCESS) {

×

5680

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

5681

    pTaskInfo->code = code;

×

5682

    T_LONG_JMP(pTaskInfo->env, code);

×

5684

  setStreamOperatorCompleted(pOperator);

×

5685

  (*ppRes) = NULL;

×

5686

  return code;

×

5689

int32_t createStreamSingleIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,

×

5692

  QRY_PARAM_CHECK(pOptrInfo);

×

5694

  int32_t code = TSDB_CODE_SUCCESS;

×

5695

  int32_t lino = 0;

×

5696

  int32_t numOfCols = 0;

×

5698

  SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo));

×

5699

  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));

×

5700

  if (pInfo == NULL || pOperator == NULL) {

×

5701

    code = terrno;

×

5702

    QUERY_CHECK_CODE(code, lino, _error);

×

5705

  SStreamIntervalPhysiNode* pIntervalPhyNode = (SStreamIntervalPhysiNode*)pPhyNode;

×

5707

  SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);

×

5708

  QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);

×

5709

  initBasicInfo(&pInfo->binfo, pResBlock);

×

5711

  pInfo->interval = (SInterval){

×

5712

      .interval = pIntervalPhyNode->interval,

×

5713

      .sliding = pIntervalPhyNode->sliding,

×

5714

      .intervalUnit = pIntervalPhyNode->intervalUnit,

×

5715

      .slidingUnit = pIntervalPhyNode->slidingUnit,

×

5716

      .offset = pIntervalPhyNode->offset,

×

5717

      .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision,

×

5718

      .timeRange = pIntervalPhyNode->timeRange,

×

5720

  calcIntervalAutoOffset(&pInfo->interval);

×

5722

  pInfo->twAggSup =

×

5723

      (STimeWindowAggSupp){.waterMark = pIntervalPhyNode->window.watermark,

×

5724

                           .calTrigger = pIntervalPhyNode->window.triggerType,

×

5727

                           .deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval)};

×

5729

  pOperator->pTaskInfo = pTaskInfo;

×

5730

  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;

×

5732

  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;

×

5733

  pInfo->ignoreExpiredDataSaved = false;

×

5735

  SExprSupp* pSup = &pOperator->exprSupp;

×

5736

  pSup->hasWindowOrGroup = true;

×

5738

  code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

×

5739

  QUERY_CHECK_CODE(code, lino, _error);

×

5741

  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;

×

5742

  initResultSizeInfo(&pOperator->resultInfo, 4096);

×

5744

  pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState));

×

5745

  QUERY_CHECK_NULL(pInfo->pState, code, lino, _error, terrno);

×

5746

  *(pInfo->pState) = *(pTaskInfo->streamInfo.pState);

×

5747

  pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsIndex);

×

5749

  size_t     keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;

×

5750

  SExprInfo* pExprInfo = NULL;

×

5751

  code = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);

×

5752

  QUERY_CHECK_CODE(code, lino, _error);

×

5753

  code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, pInfo->pState,

×

5755

  QUERY_CHECK_CODE(code, lino, _error);

×

5757

  if (pIntervalPhyNode->window.pExprs != NULL) {

×

5758

    int32_t    numOfScalar = 0;

×

5759

    SExprInfo* pScalarExprInfo = NULL;

×

5761

    code = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalar);

×

5762

    QUERY_CHECK_CODE(code, lino, _error);

×

5764

    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);

×

5765

    QUERY_CHECK_CODE(code, lino, _error);

×

5768

  pInfo->invertible = false;

×

5769

  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));

×

5770

  QUERY_CHECK_NULL(pInfo->pDelWins, code, lino, _error, terrno);

×

5771

  pInfo->delIndex = 0;

×

5773

  code = createSpecialDataBlock(STREAM_DELETE_RESULT, &pInfo->pDelRes);

×

5774

  QUERY_CHECK_CODE(code, lino, _error);

×

5776

  initResultRowInfo(&pInfo->binfo.resultRowInfo);

×

5778

  pInfo->pPhyNode = NULL;  // create new child

×

5779

  pInfo->pPullDataMap = NULL;

×

5780

  pInfo->pFinalPullDataMap = NULL;

×

5781

  pInfo->pPullWins = NULL;  // SPullWindowInfo

×

5782

  pInfo->pullIndex = 0;

×

5783

  pInfo->pPullDataRes = NULL;

×

5784

  pInfo->numOfChild = 0;

×

5785

  pInfo->delKey.ts = INT64_MAX;

×

5786

  pInfo->delKey.groupId = 0;

×

5787

  pInfo->numOfDatapack = 0;

×

5788

  pInfo->pUpdated = NULL;

×

5789

  pInfo->pUpdatedMap = NULL;

×

5790

  int32_t funResSize = getMaxFunResSize(pSup, numOfCols);

×

5792

  pInfo->stateStore = pTaskInfo->storageAPI.stateStore;

×

5793

  pInfo->pState->pFileState = NULL;

×

5796

  pInfo->pState->pResultRowStore.resultRowGet = getResultRowFromBuf;

×

5797

  pInfo->pState->pResultRowStore.resultRowPut = putResultRowToBuf;

×

5798

  pInfo->pState->pExprSupp = &pOperator->exprSupp;

×

5800

  code = pTaskInfo->storageAPI.stateStore.streamFileStateInit(

×

5801

      tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, compareTs, pInfo->pState,

×

5802

      pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo), pHandle->checkpointId, STREAM_STATE_BUFF_HASH,

×

5803

      &pInfo->pState->pFileState);

×

5804

  QUERY_CHECK_CODE(code, lino, _error);

×

5806

  pInfo->pOperator = pOperator;

×

5807

  setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED,

×

5810

      createOperatorFpSet(optrDummyOpenFn, doStreamIntervalAggNext, NULL, destroyStreamFinalIntervalOperatorInfo,

×

5812

  setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState);

×

5814

  code = initStreamBasicInfo(&pInfo->basic, pOperator);

×

5815

  QUERY_CHECK_CODE(code, lino, _error);

×

5817

  pInfo->recvGetAll = false;

×

5819

  code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);

×

5820

  QUERY_CHECK_CODE(code, lino, _error);

×

5822

  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

5823

  pInfo->pDeletedMap = tSimpleHashInit(4096, hashFn);

×

5824

  QUERY_CHECK_NULL(pInfo->pDeletedMap, code, lino, _error, terrno);

×

5825

  pInfo->destHasPrimaryKey = pIntervalPhyNode->window.destHasPrimaryKey;

×

5828

  void*   buff = NULL;

×

5829

  int32_t len = 0;

×

5830

  int32_t res = pAPI->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_CHECKPOINT_NAME,

×

5832

  if (res == TSDB_CODE_SUCCESS) {

×

5833

    doStreamIntervalDecodeOpState(buff, len, pOperator);

×

5834

    taosMemoryFree(buff);

×

5837

  pInfo->basic.primaryPkIndex = -1;

×

5838

  code = initIntervalDownStream(downstream, pPhyNode->type, pInfo, &pInfo->basic);

×

5839

  QUERY_CHECK_CODE(code, lino, _error);

×

5841

  code = appendDownstream(pOperator, &downstream, 1);

×

5842

  QUERY_CHECK_CODE(code, lino, _error);

×

5844

  *pOptrInfo = pOperator;

×

5845

  return TSDB_CODE_SUCCESS;

×

5847

_error:

×

5848

  if (pInfo != NULL) destroyStreamFinalIntervalOperatorInfo(pInfo);

×

5849

  destroyOperatorAndDownstreams(pOperator, &downstream, 1);

×

5850

  pTaskInfo->code = code;

×

5851

  return code;

×

5854

static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pUpdatedMap) {

×

5855

  int32_t                      code = TSDB_CODE_SUCCESS;

×

5856

  int32_t                      lino = 0;

×

5857

  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info;

×

5858

  pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);

×

5860

  SResultRowInfo*  pResultRowInfo = &(pInfo->binfo.resultRowInfo);

×

5861

  SExecTaskInfo*   pTaskInfo = pOperator->pTaskInfo;

×

5862

  SExprSupp*       pSup = &pOperator->exprSupp;

×

5863

  int32_t          numOfOutput = pSup->numOfExprs;

×

5864

  int32_t          step = 1;

×

5865

  SRowBuffPos*     pResPos = NULL;

×

5866

  SResultRow*      pResult = NULL;

×

5867

  int32_t          forwardRows = 1;

×

5868

  uint64_t         groupId = pSDataBlock->info.id.groupId;

×

5869

  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);

×

5870

  TSKEY*           tsCol = (int64_t*)pColDataInfo->pData;

×

5872

  int32_t     startPos = 0;

×

5873

  TSKEY       ts = getStartTsKey(&pSDataBlock->info.window, tsCol);

×

5874

  STimeWindow nextWin = getFinalTimeWindow(ts, &pInfo->interval);

×

5876

  while (1) {

×

5877

    SWinKey key = {

×

5878

        .ts = nextWin.skey,

×

5881

    void*   chIds = taosHashGet(pInfo->pPullDataMap, &key, sizeof(SWinKey));

×

5882

    int32_t index = -1;

×

5883

    SArray* chArray = NULL;

×

5884

    int32_t chId = 0;

×

5885

    if (chIds) {

×

5886

      chArray = *(void**)chIds;

×

5887

      chId = getChildIndex(pSDataBlock);

×

5888

      index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);

×

5890

    if (!(index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA)) {

×

5891

      startPos = getNextQualifiedFinalWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCol, startPos);

×

5892

      if (startPos < 0) {

×

5893

        break;

×

5895

      continue;

×

5898

    if (!inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) {

×

5899

      startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCol, startPos, nextWin.ekey, &nextWin);

×

5900

      if (startPos < 0) {

×

5901

        break;

×

5903

      continue;

×

5906

    int32_t winCode = TSDB_CODE_SUCCESS;

×

5907

    code = setIntervalOutputBuf(pInfo->pState, &nextWin, &pResPos, groupId, pSup->pCtx, numOfOutput,

×

5909

    QUERY_CHECK_CODE(code, lino, _end);

×

5911

    pResult = (SResultRow*)pResPos->pRowBuff;

×

5913

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {

×

5914

      code = saveWinResult(&key, pResPos, pUpdatedMap);

×

5915

      QUERY_CHECK_CODE(code, lino, _end);

×

5918

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

5919

      code = tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pResPos, POINTER_BYTES);

×

5920

      QUERY_CHECK_CODE(code, lino, _end);

×

5923

    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, 1);

×

5924

    code = applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos,

×

5925

                                           forwardRows, pSDataBlock->info.rows, numOfOutput);

×

5926

    QUERY_CHECK_CODE(code, lino, _end);

×

5927

    key.ts = nextWin.skey;

×

5929

    if (pInfo->delKey.ts > key.ts) {

×

5930

      pInfo->delKey = key;

×

5932

    int32_t prevEndPos = (forwardRows - 1) * step + startPos;

×

5933

    if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) {

×

5934

      qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64

×

5937

      code = blockDataUpdateTsWindow(pSDataBlock, 0);

×

5938

      QUERY_CHECK_CODE(code, lino, _end);

×

5941

      if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) {

×

5942

        qError("table uid %" PRIu64 " data block timestamp is out of range! minKey %" PRId64 ",maxKey %" PRId64,

×

5946

    startPos = getNextQualifiedFinalWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCol, prevEndPos);

×

5947

    if (startPos < 0) {

×

5948

      break;

×

5952

_end:

×

5953

  if (code != TSDB_CODE_SUCCESS) {

×

5954

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

5956

×

5958

static int32_t addMidRetriveWindow(SArray* wins, SHashObj* pMidPullMap, int32_t numOfChild) {

×

5959

  int32_t code = TSDB_CODE_SUCCESS;

×

5960

  int32_t lino = 0;

×

5961

  int32_t size = taosArrayGetSize(wins);

×

5962

  for (int32_t i = 0; i < size; i++) {

×

5963

    SWinKey* winKey = taosArrayGet(wins, i);

×

5964

    void*    chIds = taosHashGet(pMidPullMap, winKey, sizeof(SWinKey));

×

5965

    if (!chIds) {

×

5966

      code = addPullWindow(pMidPullMap, winKey, numOfChild);

×

5967

      qDebug("===stream===prepare mid operator retrive for delete %" PRId64 ", size:%d", winKey->ts, numOfChild);

×

5968

      QUERY_CHECK_CODE(code, lino, _end);

×

5971

_end:

×

5972

  if (code != TSDB_CODE_SUCCESS) {

×

5973

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

5975

  return code;

×

5978

static SSDataBlock* buildMidIntervalResult(SOperatorInfo* pOperator) {

×

5979

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

5980

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

5981

  uint16_t                     opType = pOperator->operatorType;

×

5983

  if (pInfo->recvPullover) {

×

5984

    pInfo->recvPullover = false;

×

5985

    printDataBlock(pInfo->pMidPulloverRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5986

    return pInfo->pMidPulloverRes;

×

5989

  qDebug("===stream=== build mid interval result");

×

5990

  doBuildDeleteResult(pTaskInfo, pInfo->pMidPullDatas, &pInfo->midDelIndex, pInfo->pDelRes);

×

5991

  if (pInfo->pDelRes->info.rows != 0) {

×

5993

    printDataBlock(pInfo->pDelRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));

×

5994

    return pInfo->pDelRes;

×

5997

  if (pInfo->recvRetrive) {

×

5998

    pInfo->recvRetrive = false;

×

5999

    printDataBlock(pInfo->pMidRetriveRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

6000

    return pInfo->pMidRetriveRes;

×

6003

  return NULL;

×

6006

static int32_t doStreamMidIntervalAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

6007

  int32_t                      code = TSDB_CODE_SUCCESS;

×

6008

  int32_t                      lino = 0;

×

6009

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

6010

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

6011

  SStorageAPI*                 pAPI = &pOperator->pTaskInfo->storageAPI;

×

6012

  SOperatorInfo*               downstream = pOperator->pDownstream[0];

×

6013

  SExprSupp*                   pSup = &pOperator->exprSupp;

×

6015

  qDebug("stask:%s  %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);

×

6017

  if (pOperator->status == OP_EXEC_DONE) {

×

6018

    (*ppRes) = NULL;

×

6019

    return code;

×

6020

  } else if (pOperator->status == OP_RES_TO_RETURN) {

×

6021

    SSDataBlock* resBlock = NULL;

×

6022

    code = buildIntervalResult(pOperator, &resBlock);

×

6023

    QUERY_CHECK_CODE(code, lino, _end);

×

6024

    if (resBlock != NULL) {

×

6025

      (*ppRes) = resBlock;

×

6026

      return code;

×

6029

    setOperatorCompleted(pOperator);

×

6030

    clearFunctionContext(&pOperator->exprSupp);

×

6031

    clearStreamIntervalOperator(pInfo);

×

6032

    qDebug("stask:%s  ===stream===%s clear", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType));

×

6033

    (*ppRes) = NULL;

×

6034

    return code;

×

6036

    SSDataBlock* resBlock = NULL;

×

6037

    code = buildIntervalResult(pOperator, &resBlock);

×

6038

    QUERY_CHECK_CODE(code, lino, _end);

×

6039

    if (resBlock != NULL) {

×

6040

      (*ppRes) = resBlock;

×

6041

      return code;

×

6044

    resBlock = buildMidIntervalResult(pOperator);

×

6045

    if (resBlock != NULL) {

×

6046

      (*ppRes) = resBlock;

×

6047

      return code;

×

6050

    if (pInfo->clearState) {

×

6051

      pInfo->clearState = false;

×

6052

      clearFunctionContext(&pOperator->exprSupp);

×

6053

      clearStreamIntervalOperator(pInfo);

×

6057

  if (!pInfo->pUpdated) {

×

6058

    pInfo->pUpdated = taosArrayInit(4096, POINTER_BYTES);

×

6059

    QUERY_CHECK_NULL(pInfo->pUpdated, code, lino, _end, terrno);

×

6061

  if (!pInfo->pUpdatedMap) {

×

6062

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

6063

    pInfo->pUpdatedMap = tSimpleHashInit(4096, hashFn);

×

6064

    QUERY_CHECK_NULL(pInfo->pUpdatedMap, code, lino, _end, terrno);

×

6067

  while (1) {

×

6068

    if (isTaskKilled(pTaskInfo)) {

×

6069

      qInfo("===stream=== %s task is killed, code %s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code));

×

6070

      (*ppRes) = NULL;

×

6071

      return code;

×

6074

    SSDataBlock* pBlock = NULL;

×

6075

    code = downstream->fpSet.getNextFn(downstream, &pBlock);

×

6076

    QUERY_CHECK_CODE(code, lino, _end);

×

6078

    if (pBlock == NULL) {

×

6079

      pOperator->status = OP_RES_TO_RETURN;

×

6080

      qDebug("===stream===return data:%s. recv datablock num:%" PRIu64, getStreamOpName(pOperator->operatorType),

×

6082

      pInfo->numOfDatapack = 0;

×

6083

      break;

×

6085

    pInfo->numOfDatapack++;

×

6086

    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));

×

6087

    setStreamOperatorState(&pInfo->basic, pBlock->info.type);

×

6089

    if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) {

×

6090

      pInfo->binfo.pRes->info.type = pBlock->info.type;

×

6091

    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||

×

6092

               pBlock->info.type == STREAM_CLEAR) {

×

6093

      SArray* delWins = taosArrayInit(8, sizeof(SWinKey));

×

6094

      if (!delWins) {

×

6095

        code = terrno;

×

6096

        QUERY_CHECK_CODE(code, lino, _end);

×

6099

          doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, pInfo->pFinalPullDataMap);

×

6100

      QUERY_CHECK_CODE(code, lino, _end);

×

6102

      removeResults(delWins, pInfo->pUpdatedMap);

×

6103

      void* tmp = taosArrayAddAll(pInfo->pDelWins, delWins);

×

6104

      if (!tmp && taosArrayGetSize(delWins) > 0) {

×

6105

        code = TSDB_CODE_OUT_OF_MEMORY;

×

6106

        QUERY_CHECK_CODE(code, lino, _end);

×

6108

      taosArrayDestroy(delWins);

×

6110

      doBuildDeleteResult(pTaskInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);

×

6111

      if (pInfo->pDelRes->info.rows != 0) {

×

6113

        printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

6114

        if (pBlock->info.type == STREAM_CLEAR) {

×

6115

          pInfo->pDelRes->info.type = STREAM_CLEAR;

×

6117

          pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;

×

6119

        (*ppRes) = pInfo->pDelRes;

×

6120

        return code;

×

6122

      continue;

×

6123

    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {

×

6124

      (*ppRes) = pBlock;

×

6125

      return code;

×

6126

    } else if (pBlock->info.type == STREAM_PULL_OVER) {

×

6127

      code = processPullOver(pBlock, pInfo->pPullDataMap, pInfo->pFinalPullDataMap, &pInfo->interval, pInfo->pPullWins,

×

6129

      QUERY_CHECK_CODE(code, lino, _end);

×

6131

      if (pInfo->recvPullover) {

×

6132

        code = copyDataBlock(pInfo->pMidPulloverRes, pBlock);

×

6133

        QUERY_CHECK_CODE(code, lino, _end);

×

6135

        pInfo->clearState = true;

×

6136

        break;

×

6138

      continue;

×

6139

    } else if (pBlock->info.type == STREAM_CHECKPOINT) {

×

6140

      pAPI->stateStore.streamStateCommit(pInfo->pState);

×

6141

      doStreamIntervalSaveCheckpoint(pOperator);

×

6142

      code = copyDataBlock(pInfo->pCheckpointRes, pBlock);

×

6143

      QUERY_CHECK_CODE(code, lino, _end);

×

6145

      continue;

×

6146

    } else if (pBlock->info.type == STREAM_MID_RETRIEVE) {

×

6147

      SArray* delWins = taosArrayInit(8, sizeof(SWinKey));

×

6148

      if (!delWins) {

×

6149

        code = terrno;

×

6150

        QUERY_CHECK_CODE(code, lino, _end);

×

6152

      code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, NULL);

×

6153

      QUERY_CHECK_CODE(code, lino, _end);

×

6155

      code = addMidRetriveWindow(delWins, pInfo->pPullDataMap, pInfo->numOfChild);

×

6156

      QUERY_CHECK_CODE(code, lino, _end);

×

6158

      taosArrayDestroy(delWins);

×

6159

      pInfo->recvRetrive = true;

×

6160

      code = copyDataBlock(pInfo->pMidRetriveRes, pBlock);

×

6161

      QUERY_CHECK_CODE(code, lino, _end);

×

6163

      pInfo->pMidRetriveRes->info.type = STREAM_MID_RETRIEVE;

×

6164

      pInfo->clearState = true;

×

6165

      break;

×

6167

      if (pBlock->info.type != STREAM_INVALID) {

×

6168

        code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;

×

6169

        QUERY_CHECK_CODE(code, lino, _end);

×

6173

    if (pInfo->scalarSupp.pExprInfo != NULL) {

×

6174

      SExprSupp* pExprSup = &pInfo->scalarSupp;

×

6175

      code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);

×

6176

      QUERY_CHECK_CODE(code, lino, _end);

×

6178

    code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

6179

    QUERY_CHECK_CODE(code, lino, _end);

×

6180

    doStreamMidIntervalAggImpl(pOperator, pBlock, pInfo->pUpdatedMap);

×

6181

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);

×

6182

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.watermark);

×

6183

    pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey);

×

6186

  removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins);

×

6187

  pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs;

×

6189

  void*   pIte = NULL;

×

6190

  int32_t iter = 0;

×

6191

  while ((pIte = tSimpleHashIterate(pInfo->pUpdatedMap, pIte, &iter)) != NULL) {

×

6192

    void* tmp = taosArrayPush(pInfo->pUpdated, pIte);

×

6193

    if (!tmp) {

×

6194

      code = terrno;

×

6195

      QUERY_CHECK_CODE(code, lino, _end);

×

6199

  tSimpleHashCleanup(pInfo->pUpdatedMap);

×

6200

  pInfo->pUpdatedMap = NULL;

×

6201

  taosArraySort(pInfo->pUpdated, winPosCmprImpl);

×

6203

  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);

×

6204

  pInfo->pUpdated = NULL;

×

6205

  code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);

×

6206

  QUERY_CHECK_CODE(code, lino, _end);

×

6208

  SSDataBlock* resBlock = NULL;

×

6209

  code = buildIntervalResult(pOperator, &resBlock);

×

6210

  QUERY_CHECK_CODE(code, lino, _end);

×

6211

  if (resBlock != NULL) {

×

6212

    (*ppRes) = resBlock;

×

6213

    return code;

×

6216

  resBlock = buildMidIntervalResult(pOperator);

×

6217

  if (resBlock != NULL) {

×

6218

    (*ppRes) = resBlock;

×

6219

    return code;

×

6222

  if (pInfo->clearState) {

×

6223

    pInfo->clearState = false;

×

6224

    clearFunctionContext(&pOperator->exprSupp);

×

6225

    clearStreamIntervalOperator(pInfo);

×

6228

_end:

×

6229

  if (code != TSDB_CODE_SUCCESS) {

×

6230

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

6231

    pTaskInfo->code = code;

×

6232

    T_LONG_JMP(pTaskInfo->env, code);

×

6234

  (*ppRes) = NULL;

×

6235

  return code;

×

6238

static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) {

×

6239

  SSDataBlock* pRes = NULL;

×

6240

  int32_t      code = doStreamMidIntervalAggNext(pOperator, &pRes);

×

6241

  return pRes;

×

6244

void setStreamOperatorCompleted(SOperatorInfo* pOperator) {

×

6245

  qDebug("stask:%s  %s status: %d. set completed", GET_TASKID(pOperator->pTaskInfo),

×

6247

  setOperatorCompleted(pOperator);

×

6248

×

taosdata / TDengine / #4143

Source File
Press 'n' to go to next uncovered line, 'b' for previous

taosdata / TDengine / #4143

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous