taosdata/TDengine | Build 3646 | source/libs/executor/src/streamtimewindowoperator.c | Coveralls

61

static int32_t binarySearchCom(void* keyList, int num, void* pKey, int order, __compare_fn_t comparefn) {

×

62

  int firstPos = 0, lastPos = num - 1, midPos = -1;

×

63

  int numOfRows = 0;

×

65

  if (num <= 0) return -1;

×

66

  if (order == TSDB_ORDER_DESC) {

×

69

      if (comparefn(pKey, keyList, lastPos) >= 0) return lastPos;

×

70

      if (comparefn(pKey, keyList, firstPos) == 0) return firstPos;

×

71

      if (comparefn(pKey, keyList, firstPos) < 0) return firstPos - 1;

×

73

      numOfRows = lastPos - firstPos + 1;

×

74

      midPos = (numOfRows >> 1) + firstPos;

×

76

      if (comparefn(pKey, keyList, midPos) < 0) {

×

77

        lastPos = midPos - 1;

×

78

      } else if (comparefn(pKey, keyList, midPos) > 0) {

×

79

        firstPos = midPos + 1;

×

81

        break;

×

88

      if (comparefn(pKey, keyList, firstPos) <= 0) return firstPos;

×

89

      if (comparefn(pKey, keyList, lastPos) == 0) return lastPos;

×

91

      if (comparefn(pKey, keyList, lastPos) > 0) {

×

92

        lastPos = lastPos + 1;

×

93

        if (lastPos >= num)

×

94

          return -1;

×

96

          return lastPos;

×

99

      numOfRows = lastPos - firstPos + 1;

×

100

      midPos = (numOfRows >> 1) + firstPos;

×

102

      if (comparefn(pKey, keyList, midPos) < 0) {

×

103

        lastPos = midPos - 1;

×

104

      } else if (comparefn(pKey, keyList, midPos) > 0) {

×

105

        firstPos = midPos + 1;

×

107

        break;

×

112

  return midPos;

×

115

static int32_t comparePullWinKey(void* pKey, void* data, int32_t index) {

×

116

  SArray*          res = (SArray*)data;

×

117

  SPullWindowInfo* pos = taosArrayGet(res, index);

×

118

  SPullWindowInfo* pData = (SPullWindowInfo*)pKey;

×

119

  if (pData->groupId > pos->groupId) {

×

120

    return 1;

×

121

  } else if (pData->groupId < pos->groupId) {

×

122

    return -1;

×

125

  if (pData->window.skey > pos->window.ekey) {

×

126

    return 1;

×

127

  } else if (pData->window.ekey < pos->window.skey) {

×

128

    return -1;

×

130

  return 0;

×

133

static int32_t savePullWindow(SPullWindowInfo* pPullInfo, SArray* pPullWins) {

×

134

  int32_t size = taosArrayGetSize(pPullWins);

×

135

  int32_t index = binarySearchCom(pPullWins, size, pPullInfo, TSDB_ORDER_DESC, comparePullWinKey);

×

136

  if (index == -1) {

×

137

    index = 0;

×

139

    int32_t code = comparePullWinKey(pPullInfo, pPullWins, index);

×

140

    if (code == 0) {

×

141

      SPullWindowInfo* pos = taosArrayGet(pPullWins, index);

×

142

      pos->window.skey = TMIN(pos->window.skey, pPullInfo->window.skey);

×

143

      pos->window.ekey = TMAX(pos->window.ekey, pPullInfo->window.ekey);

×

144

      pos->calWin.skey = TMIN(pos->calWin.skey, pPullInfo->calWin.skey);

×

145

      pos->calWin.ekey = TMAX(pos->calWin.ekey, pPullInfo->calWin.ekey);

×

146

      return TSDB_CODE_SUCCESS;

×

147

    } else if (code > 0) {

×

148

      index++;

×

151

  if (taosArrayInsert(pPullWins, index, pPullInfo) == NULL) {

×

152

    return terrno;

×

154

  return TSDB_CODE_SUCCESS;

×

157

int32_t saveResult(SResultWindowInfo winInfo, SSHashObj* pStUpdated) {

×

158

  if (tSimpleHashGetSize(pStUpdated) > MAX_STREAM_HISTORY_RESULT) {

×

159

    qError("%s failed at line %d since too many history result. ", __func__, __LINE__);

×

160

    return TSDB_CODE_STREAM_INTERNAL_ERROR;

×

162

  winInfo.sessionWin.win.ekey = winInfo.sessionWin.win.skey;

×

163

  return tSimpleHashPut(pStUpdated, &winInfo.sessionWin, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo));

×

166

int32_t saveWinResult(SWinKey* pKey, SRowBuffPos* pPos, SSHashObj* pUpdatedMap) {

×

167

  if (tSimpleHashGetSize(pUpdatedMap) > MAX_STREAM_HISTORY_RESULT) {

×

168

    qError("%s failed at line %d since too many history result. ", __func__, __LINE__);

×

169

    return TSDB_CODE_STREAM_INTERNAL_ERROR;

×

171

  return tSimpleHashPut(pUpdatedMap, pKey, sizeof(SWinKey), &pPos, POINTER_BYTES);

×

174

static int32_t saveWinResultInfo(TSKEY ts, uint64_t groupId, SRowBuffPos* pPos, SSHashObj* pUpdatedMap) {

×

175

  SWinKey key = {.ts = ts, .groupId = groupId};

×

176

  return saveWinResult(&key, pPos, pUpdatedMap);

×

179

static void removeResults(SArray* pWins, SSHashObj* pUpdatedMap) {

×

180

  int32_t size = taosArrayGetSize(pWins);

×

181

  for (int32_t i = 0; i < size; i++) {

×

182

    SWinKey* pW = taosArrayGet(pWins, i);

×

183

    void*    tmp = tSimpleHashGet(pUpdatedMap, pW, sizeof(SWinKey));

×

184

    if (tmp) {

×

185

      void* value = *(void**)tmp;

×

186

      taosMemoryFree(value);

×

187

      int32_t tmpRes = tSimpleHashRemove(pUpdatedMap, pW, sizeof(SWinKey));

×

188

      qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

191

×

193

static int32_t compareWinKey(void* pKey, void* data, int32_t index) {

×

194

  void* pDataPos = taosArrayGet((SArray*)data, index);

×

195

  return winKeyCmprImpl(pKey, pDataPos);

×

198

void removeDeleteResults(SSHashObj* pUpdatedMap, SArray* pDelWins) {

×

199

  taosArraySort(pDelWins, winKeyCmprImpl);

×

200

  taosArrayRemoveDuplicate(pDelWins, winKeyCmprImpl, NULL);

×

201

  int32_t delSize = taosArrayGetSize(pDelWins);

×

202

  if (tSimpleHashGetSize(pUpdatedMap) == 0 || delSize == 0) {

×

203

    return;

×

205

  void*   pIte = NULL;

×

206

  int32_t iter = 0;

×

207

  while ((pIte = tSimpleHashIterate(pUpdatedMap, pIte, &iter)) != NULL) {

×

208

    SWinKey* pResKey = tSimpleHashGetKey(pIte, NULL);

×

209

    int32_t  index = binarySearchCom(pDelWins, delSize, pResKey, TSDB_ORDER_DESC, compareWinKey);

×

210

    if (index >= 0 && 0 == compareWinKey(pResKey, pDelWins, index)) {

×

211

      taosArrayRemove(pDelWins, index);

×

212

      delSize = taosArrayGetSize(pDelWins);

×

217

bool isOverdue(TSKEY ekey, STimeWindowAggSupp* pTwSup) {

×

218

  return pTwSup->maxTs != INT64_MIN && ekey < pTwSup->maxTs - pTwSup->waterMark;

×

221

bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pTwSup) { return isOverdue(pWin->ekey, pTwSup); }

×

223

static void doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId) {

×

224

  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;

×

226

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

227

  SWinKey                      key = {.ts = ts, .groupId = groupId};

×

228

  int32_t                      tmpRes = tSimpleHashRemove(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey));

×

229

  qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

230

  pAPI->stateStore.streamStateDel(pInfo->pState, &key);

×

231

×

233

static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; }

×

235

static void doDeleteWindowByGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock) {

×

236

  SStorageAPI*                 pAPI = &pOperator->pTaskInfo->storageAPI;

×

237

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

239

  SColumnInfoData* pGpIdCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);

×

240

  uint64_t* pGroupIdData = (uint64_t*)pGpIdCol->pData;

×

241

  for (int32_t i = 0; i < pBlock->info.rows; i++) {

×

242

    uint64_t groupId = pGroupIdData[i];

×

243

    void*   pIte = NULL;

×

244

    int32_t iter = 0;

×

245

    while ((pIte = tSimpleHashIterate(pInfo->aggSup.pResultRowHashTable, pIte, &iter)) != NULL) {

×

246

      size_t keyLen = 0;

×

247

      SWinKey* pKey = tSimpleHashGetKey(pIte, &keyLen);

×

248

      if (pKey->groupId == groupId) {

×

249

        int32_t tmpRes = tSimpleHashIterateRemove(pInfo->aggSup.pResultRowHashTable, pKey, keyLen, &pIte, &iter);

×

250

        qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

254

    pAPI->stateStore.streamStateDelByGroupId(pInfo->pState, groupId);

×

256

×

258

static int32_t doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDataBlock* pBlock, SArray* pUpWins,

×

260

  int32_t                      code = TSDB_CODE_SUCCESS;

×

261

  int32_t                      lino = 0;

×

262

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

263

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

264

  SColumnInfoData*             pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);

×

265

  TSKEY*                       startTsCols = (TSKEY*)pStartTsCol->pData;

×

266

  SColumnInfoData*             pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);

×

267

  TSKEY*                       endTsCols = (TSKEY*)pEndTsCol->pData;

×

268

  SColumnInfoData*             pCalStTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);

×

269

  TSKEY*                       calStTsCols = (TSKEY*)pCalStTsCol->pData;

×

270

  SColumnInfoData*             pCalEnTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);

×

271

  TSKEY*                       calEnTsCols = (TSKEY*)pCalEnTsCol->pData;

×

272

  SColumnInfoData*             pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);

×

273

  uint64_t*                    pGpDatas = (uint64_t*)pGpCol->pData;

×

274

  for (int32_t i = 0; i < pBlock->info.rows; i++) {

×

275

    SResultRowInfo dumyInfo = {0};

×

276

    dumyInfo.cur.pageId = -1;

×

278

    STimeWindow win = {0};

×

279

    if (IS_FINAL_INTERVAL_OP(pOperator) || IS_MID_INTERVAL_OP(pOperator)) {

×

280

      win.skey = startTsCols[i];

×

281

      win.ekey = endTsCols[i];

×

283

      win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC);

×

287

      if (!inCalSlidingWindow(pInterval, &win, calStTsCols[i], calEnTsCols[i], pBlock->info.type)) {

×

288

        getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC);

×

289

        continue;

×

291

      uint64_t winGpId = pGpDatas[i];

×

292

      SWinKey  winRes = {.ts = win.skey, .groupId = winGpId};

×

293

      void*    chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey));

×

294

      if (chIds) {

×

295

        int32_t childId = getChildIndex(pBlock);

×

296

        if (pInvalidWins) {

×

297

          qDebug("===stream===save invalid delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d", winRes.ts,

×

299

          code = taosHashPut(pInvalidWins, &winRes, sizeof(SWinKey), NULL, 0);

×

300

          QUERY_CHECK_CODE(code, lino, _end);

×

303

        SArray* chArray = *(void**)chIds;

×

304

        int32_t index = taosArraySearchIdx(chArray, &childId, compareInt32Val, TD_EQ);

×

305

        if (index != -1) {

×

306

          qDebug("===stream===try push delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d ,continue", win.skey,

×

308

          getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC);

×

309

          continue;

×

312

      doDeleteWindow(pOperator, win.skey, winGpId);

×

313

      if (pUpWins) {

×

314

        void* tmp = taosArrayPush(pUpWins, &winRes);

×

315

        if (!tmp) {

×

316

          code = terrno;

×

317

          QUERY_CHECK_CODE(code, lino, _end);

×

320

      if (pUpdatedMap) {

×

321

        int32_t tmpRes = tSimpleHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey));

×

322

        qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

324

      getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC);

×

325

    } while (win.ekey <= endTsCols[i]);

×

327

_end:

×

328

  if (code != TSDB_CODE_SUCCESS) {

×

329

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

331

  return code;

×

334

static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SSHashObj* resWins) {

×

335

  void*   pIte = NULL;

×

336

  int32_t iter = 0;

×

337

  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {

×

338

    SWinKey*     pKey = tSimpleHashGetKey(pIte, NULL);

×

339

    uint64_t     groupId = pKey->groupId;

×

340

    TSKEY        ts = pKey->ts;

×

341

    SRowBuffPos* pPos = *(SRowBuffPos**)pIte;

×

342

    if (!pPos->beUpdated) {

×

343

      continue;

×

345

    pPos->beUpdated = false;

×

346

    int32_t code = saveWinResultInfo(ts, groupId, pPos, resWins);

×

347

    if (code != TSDB_CODE_SUCCESS) {

×

348

      return code;

×

351

  return TSDB_CODE_SUCCESS;

×

354

static int32_t closeStreamIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SInterval* pInterval,

×

357

  qDebug("===stream===close interval window");

×

358

  int32_t                      code = TSDB_CODE_SUCCESS;

×

359

  int32_t                      lino = 0;

×

360

  void*                        pIte = NULL;

×

361

  int32_t                      iter = 0;

×

362

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

363

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

364

  int32_t                      delSize = taosArrayGetSize(pDelWins);

×

365

  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {

×

366

    void*    key = tSimpleHashGetKey(pIte, NULL);

×

367

    SWinKey* pWinKey = (SWinKey*)key;

×

368

    if (delSize > 0) {

×

369

      int32_t index = binarySearchCom(pDelWins, delSize, pWinKey, TSDB_ORDER_DESC, compareWinKey);

×

370

      if (index >= 0 && 0 == compareWinKey(pWinKey, pDelWins, index)) {

×

371

        taosArrayRemove(pDelWins, index);

×

372

        delSize = taosArrayGetSize(pDelWins);

×

376

    void*       chIds = taosHashGet(pPullDataMap, pWinKey, sizeof(SWinKey));

×

377

    STimeWindow win = {

×

378

        .skey = pWinKey->ts,

×

379

        .ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision, NULL) - 1,

×

381

    if (isCloseWindow(&win, pTwSup)) {

×

382

      if (chIds && pPullDataMap) {

×

383

        SArray* chAy = *(SArray**)chIds;

×

384

        int32_t size = taosArrayGetSize(chAy);

×

385

        qDebug("===stream===window %" PRId64 " wait child size:%d", pWinKey->ts, size);

×

386

        for (int32_t i = 0; i < size; i++) {

×

387

          qDebug("===stream===window %" PRId64 " wait child id:%d", pWinKey->ts, *(int32_t*)taosArrayGet(chAy, i));

×

389

        continue;

×

390

      } else if (pPullDataMap) {

×

391

        qDebug("===stream===close window %" PRId64, pWinKey->ts);

×

394

      if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

395

        code = saveWinResult(pWinKey, *(SRowBuffPos**)pIte, closeWins);

×

396

        QUERY_CHECK_CODE(code, lino, _end);

×

398

      int32_t tmpRes = tSimpleHashIterateRemove(pHashMap, pWinKey, sizeof(SWinKey), &pIte, &iter);

×

399

      qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

403

_end:

×

404

  if (code != TSDB_CODE_SUCCESS) {

×

405

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

407

  return code;

×

410

STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) {

×

411

  STimeWindow w = {.skey = ts, .ekey = INT64_MAX};

×

412

  w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision, NULL) - 1;

×

413

  return w;

×

416

static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWins, int32_t* index,

×

418

  doBuildDeleteResultImpl(&pInfo->stateStore, pInfo->pState, pWins, index, pBlock);

×

419

×

421

void doBuildDeleteResultImpl(SStateStore* pAPI, SStreamState* pState, SArray* pWins, int32_t* index,

×

423

  int32_t code = TSDB_CODE_SUCCESS;

×

424

  int32_t lino = 0;

×

425

  blockDataCleanup(pBlock);

×

426

  int32_t size = taosArrayGetSize(pWins);

×

427

  if (*index == size) {

×

428

    *index = 0;

×

429

    taosArrayClear(pWins);

×

430

    goto _end;

×

432

  code = blockDataEnsureCapacity(pBlock, size - *index);

×

433

  QUERY_CHECK_CODE(code, lino, _end);

×

435

  uint64_t uid = 0;

×

436

  for (int32_t i = *index; i < size; i++) {

×

437

    SWinKey* pWin = taosArrayGet(pWins, i);

×

438

    void*    tbname = NULL;

×

439

    int32_t  winCode = TSDB_CODE_SUCCESS;

×

440

    code = pAPI->streamStateGetParName(pState, pWin->groupId, &tbname, false, &winCode);

×

441

    QUERY_CHECK_CODE(code, lino, _end);

×

443

    if (winCode != TSDB_CODE_SUCCESS) {

×

444

      code = appendDataToSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, NULL);

×

445

      QUERY_CHECK_CODE(code, lino, _end);

×

447

      QUERY_CHECK_CONDITION((tbname), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);

×

449

      STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));

×

450

      code = appendDataToSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, parTbName);

×

451

      QUERY_CHECK_CODE(code, lino, _end);

×

453

    pAPI->streamStateFreeVal(tbname);

×

454

    (*index)++;

×

457

_end:

×

458

  if (code != TSDB_CODE_SUCCESS) {

×

459

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

461

×

463

void destroyFlusedPos(void* pRes) {

×

464

  SRowBuffPos* pPos = (SRowBuffPos*)pRes;

×

465

  if (pPos->needFree && !pPos->pRowBuff) {

×

466

    taosMemoryFreeClear(pPos->pKey);

×

467

    taosMemoryFree(pPos);

×

469

×

471

void destroyFlusedppPos(void* ppRes) {

×

472

  void* pRes = *(void**)ppRes;

×

473

  destroyFlusedPos(pRes);

×

474

×

476

void clearGroupResInfo(SGroupResInfo* pGroupResInfo) {

×

477

  int32_t size = taosArrayGetSize(pGroupResInfo->pRows);

×

478

  if (pGroupResInfo->index >= 0 && pGroupResInfo->index < size) {

×

479

    for (int32_t i = pGroupResInfo->index; i < size; i++) {

×

480

      void* pPos = taosArrayGetP(pGroupResInfo->pRows, i);

×

481

      destroyFlusedPos(pPos);

×

484

  pGroupResInfo->freeItem = false;

×

485

  taosArrayDestroy(pGroupResInfo->pRows);

×

486

  pGroupResInfo->pRows = NULL;

×

487

  pGroupResInfo->index = 0;

×

488

  pGroupResInfo->delIndex = 0;

×

489

×

491

void destroyStreamFinalIntervalOperatorInfo(void* param) {

×

492

  if (param == NULL) {

×

493

    return;

×

495

  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)param;

×

496

  cleanupBasicInfo(&pInfo->binfo);

×

497

  if (pInfo->pOperator) {

×

498

    cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, &pInfo->groupResInfo, &pInfo->aggSup,

×

500

    pInfo->pOperator = NULL;

×

503

  destroyStreamBasicInfo(&pInfo->basic);

×

504

  cleanupAggSup(&pInfo->aggSup);

×

505

  clearGroupResInfo(&pInfo->groupResInfo);

×

506

  taosArrayDestroyP(pInfo->pUpdated, destroyFlusedPos);

×

507

  pInfo->pUpdated = NULL;

×

510

  void* pIte = NULL;

×

511

  while ((pIte = taosHashIterate(pInfo->pPullDataMap, pIte)) != NULL) {

×

512

    taosArrayDestroy(*(void**)pIte);

×

514

  taosHashCleanup(pInfo->pPullDataMap);

×

515

  taosHashCleanup(pInfo->pFinalPullDataMap);

×

516

  taosArrayDestroy(pInfo->pPullWins);

×

517

  blockDataDestroy(pInfo->pPullDataRes);

×

518

  taosArrayDestroy(pInfo->pDelWins);

×

519

  blockDataDestroy(pInfo->pDelRes);

×

520

  blockDataDestroy(pInfo->pMidRetriveRes);

×

521

  blockDataDestroy(pInfo->pMidPulloverRes);

×

522

  if (pInfo->pUpdatedMap != NULL) {

×

524

    tSimpleHashSetFreeFp(pInfo->pUpdatedMap, destroyFlusedppPos);

×

525

    tSimpleHashCleanup(pInfo->pUpdatedMap);

×

526

    pInfo->pUpdatedMap = NULL;

×

529

  if (pInfo->stateStore.streamFileStateDestroy != NULL) {

×

530

    pInfo->stateStore.streamFileStateDestroy(pInfo->pState->pFileState);

×

532

  taosArrayDestroy(pInfo->pMidPullDatas);

×

534

  if (pInfo->pState != NULL && pInfo->pState->dump == 1) {

×

535

    taosMemoryFreeClear(pInfo->pState->pTdbState->pOwner);

×

536

    taosMemoryFreeClear(pInfo->pState->pTdbState);

×

538

  taosMemoryFreeClear(pInfo->pState);

×

540

  nodesDestroyNode((SNode*)pInfo->pPhyNode);

×

541

  colDataDestroy(&pInfo->twAggSup.timeWindowData);

×

542

  cleanupExprSupp(&pInfo->scalarSupp);

×

543

  tSimpleHashCleanup(pInfo->pDeletedMap);

×

545

  blockDataDestroy(pInfo->pCheckpointRes);

×

547

  taosMemoryFreeClear(param);

×

561

void reloadFromDownStream(SOperatorInfo* downstream, SStreamIntervalOperatorInfo* pInfo) {

×

562

  SStateStore* pAPI = &downstream->pTaskInfo->storageAPI.stateStore;

×

564

  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {

×

565

    reloadFromDownStream(downstream->pDownstream[0], pInfo);

×

566

    return;

×

569

  SStreamScanInfo* pScanInfo = downstream->info;

×

570

  pInfo->pUpdateInfo = pScanInfo->pUpdateInfo;

×

573

bool hasSrcPrimaryKeyCol(SSteamOpBasicInfo* pInfo) { return pInfo->primaryPkIndex != -1; }

×

575

int32_t initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamIntervalOperatorInfo* pInfo,

×

577

  int32_t        code = TSDB_CODE_SUCCESS;

×

578

  int32_t        lino = 0;

×

579

  SStateStore*   pAPI = &downstream->pTaskInfo->storageAPI.stateStore;

×

580

  SExecTaskInfo* pTaskInfo = downstream->pTaskInfo;

×

582

  if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) {

×

583

    SStreamPartitionOperatorInfo* pScanInfo = downstream->info;

×

584

    pBasic->primaryPkIndex = pScanInfo->basic.primaryPkIndex;

×

587

  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {

×

588

    return initIntervalDownStream(downstream->pDownstream[0], type, pInfo, pBasic);

×

591

  SStreamScanInfo* pScanInfo = downstream->info;

×

592

  pScanInfo->windowSup.parentType = type;

×

593

  pScanInfo->windowSup.pIntervalAggSup = &pInfo->aggSup;

×

594

  if (!pScanInfo->pUpdateInfo) {

×

595

    code = pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate,

×

596

                                 pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo);

×

597

    QUERY_CHECK_CODE(code, lino, _end);

×

600

  pScanInfo->interval = pInfo->interval;

×

601

  pScanInfo->twAggSup = pInfo->twAggSup;

×

602

  pScanInfo->pState = pInfo->pState;

×

603

  pInfo->pUpdateInfo = pScanInfo->pUpdateInfo;

×

604

  if (!hasSrcPrimaryKeyCol(pBasic)) {

×

605

    pBasic->primaryPkIndex = pScanInfo->basic.primaryPkIndex;

×

608

_end:

×

609

  if (code != TSDB_CODE_SUCCESS) {

×

610

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

612

  return code;

×

615

int32_t compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput,

×

617

  int32_t code = TSDB_CODE_SUCCESS;

×

618

  int32_t lino = 0;

×

619

  for (int32_t k = 0; k < numOfOutput; ++k) {

×

620

    if (fmIsWindowPseudoColumnFunc(pDestCtx[k].functionId)) {

×

621

      if (!pTimeWindowData) {

×

622

        continue;

×

625

      SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pDestCtx[k]);

×

626

      char*                p = GET_ROWCELL_INTERBUF(pEntryInfo);

×

627

      SColumnInfoData      idata = {0};

×

628

      idata.info.type = TSDB_DATA_TYPE_BIGINT;

×

629

      idata.info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes;

×

630

      idata.pData = p;

×

632

      SScalarParam out = {.columnData = &idata};

×

633

      SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData};

×

634

      code = pDestCtx[k].sfp.process(&tw, 1, &out);

×

635

      QUERY_CHECK_CODE(code, lino, _end);

×

637

      pEntryInfo->numOfRes = 1;

×

638

    } else if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) {

×

639

      code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]);

×

640

      QUERY_CHECK_CODE(code, lino, _end);

×

641

    } else if (pDestCtx[k].fpSet.combine == NULL) {

×

642

      char* funName = fmGetFuncName(pDestCtx[k].functionId);

×

643

      qError("%s error, combine funcion for %s is not implemented", GET_TASKID(pTaskInfo), funName);

×

644

      taosMemoryFreeClear(funName);

×

645

      code = TSDB_CODE_FAILED;

×

646

      QUERY_CHECK_CODE(code, lino, _end);

×

650

_end:

×

651

  if (code != TSDB_CODE_SUCCESS) {

×

652

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

654

  return code;

×

657

bool hasIntervalWindow(void* pState, SWinKey* pKey, SStateStore* pStore) {

×

658

  return pStore->streamStateCheck(pState, pKey);

×

661

int32_t setIntervalOutputBuf(void* pState, STimeWindow* win, SRowBuffPos** pResult, int64_t groupId,

×

664

  int32_t code = TSDB_CODE_SUCCESS;

×

665

  int32_t lino = 0;

×

666

  SWinKey key = {.ts = win->skey, .groupId = groupId};

×

667

  char*   value = NULL;

×

668

  int32_t size = pAggSup->resultRowSize;

×

670

  code = pStore->streamStateAddIfNotExist(pState, &key, (void**)&value, &size, pWinCode);

×

671

  QUERY_CHECK_CODE(code, lino, _end);

×

673

  *pResult = (SRowBuffPos*)value;

×

674

  SResultRow* res = (SResultRow*)((*pResult)->pRowBuff);

×

677

  res->win = (*win);

×

678

  code = setResultRowInitCtx(res, pCtx, numOfOutput, rowEntryInfoOffset);

×

679

  QUERY_CHECK_CODE(code, lino, _end);

×

681

_end:

×

682

  if (code != TSDB_CODE_SUCCESS) {

×

683

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

685

  return code;

×

688

bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, STimeWindowAggSupp* pTwSup,

×

690

  if (pTwSup->maxTs != INT64_MIN && pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark) {

×

691

    SWinKey key = {.ts = pWin->skey, .groupId = groupId};

×

692

    if (!hasIntervalWindow(pState, &key, pStore)) {

×

693

      return true;

×

695

    return false;

×

697

  return false;

×

700

int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols, int32_t startPos, TSKEY eKey,

×

703

      getNumOfRowsInTimeWindow(pBlockInfo, tsCols, startPos, eKey, binarySearchForKey, NULL, TSDB_ORDER_ASC);

×

704

  int32_t prevEndPos = forwardRows - 1 + startPos;

×

705

  return getNextQualifiedWindow(pInterval, pNextWin, pBlockInfo, tsCols, prevEndPos, TSDB_ORDER_ASC);

×

708

int32_t addPullWindow(SHashObj* pMap, SWinKey* pWinRes, int32_t size) {

×

709

  int32_t code = TSDB_CODE_SUCCESS;

×

710

  int32_t lino = 0;

×

711

  SArray* childIds = taosArrayInit(8, sizeof(int32_t));

×

712

  QUERY_CHECK_NULL(childIds, code, lino, _end, terrno);

×

713

  for (int32_t i = 0; i < size; i++) {

×

714

    void* tmp = taosArrayPush(childIds, &i);

×

715

    if (!tmp) {

×

716

      code = terrno;

×

717

      QUERY_CHECK_CODE(code, lino, _end);

×

720

  code = taosHashPut(pMap, pWinRes, sizeof(SWinKey), &childIds, sizeof(void*));

×

721

  QUERY_CHECK_CODE(code, lino, _end);

×

722

_end:

×

723

  if (code != TSDB_CODE_SUCCESS) {

×

724

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

726

  return code;

×

729

static void clearStreamIntervalOperator(SStreamIntervalOperatorInfo* pInfo) {

×

730

  tSimpleHashClear(pInfo->aggSup.pResultRowHashTable);

×

731

  clearDiskbasedBuf(pInfo->aggSup.pResultBuf);

×

732

  initResultRowInfo(&pInfo->binfo.resultRowInfo);

×

733

  pInfo->aggSup.currentPageId = -1;

×

734

  pInfo->stateStore.streamStateClear(pInfo->pState);

×

735

×

737

static void clearSpecialDataBlock(SSDataBlock* pBlock) {

×

738

  if (pBlock->info.rows <= 0) {

×

739

    return;

×

741

  blockDataCleanup(pBlock);

×

744

static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pBlock) {

×

745

  int32_t code = TSDB_CODE_SUCCESS;

×

746

  int32_t lino = 0;

×

747

  clearSpecialDataBlock(pBlock);

×

748

  int32_t size = taosArrayGetSize(array);

×

749

  if (size - (*pIndex) == 0) {

×

750

    goto _end;

×

752

  code = blockDataEnsureCapacity(pBlock, size - (*pIndex));

×

753

  QUERY_CHECK_CODE(code, lino, _end);

×

755

  SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);

×

756

  SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);

×

757

  SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);

×

758

  SColumnInfoData* pCalStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);

×

759

  SColumnInfoData* pCalEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);

×

760

  SColumnInfoData* pTbName = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);

×

761

  SColumnInfoData* pPrimaryKey = NULL;

×

762

  if (taosArrayGetSize(pBlock->pDataBlock) > PRIMARY_KEY_COLUMN_INDEX) {

×

763

    pPrimaryKey = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, PRIMARY_KEY_COLUMN_INDEX);

×

765

  for (; (*pIndex) < size; (*pIndex)++) {

×

766

    SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex));

×

767

    code = colDataSetVal(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false);

×

768

    QUERY_CHECK_CODE(code, lino, _end);

×

770

    code = colDataSetVal(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false);

×

771

    QUERY_CHECK_CODE(code, lino, _end);

×

773

    code = colDataSetVal(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false);

×

774

    QUERY_CHECK_CODE(code, lino, _end);

×

776

    code = colDataSetVal(pCalStartTs, pBlock->info.rows, (const char*)&pWin->calWin.skey, false);

×

777

    QUERY_CHECK_CODE(code, lino, _end);

×

779

    code = colDataSetVal(pCalEndTs, pBlock->info.rows, (const char*)&pWin->calWin.ekey, false);

×

780

    QUERY_CHECK_CODE(code, lino, _end);

×

782

    colDataSetNULL(pTbName, pBlock->info.rows);

×

783

    if (pPrimaryKey != NULL) {

×

784

      colDataSetNULL(pPrimaryKey, pBlock->info.rows);

×

787

    pBlock->info.rows++;

×

789

  if ((*pIndex) == size) {

×

790

    *pIndex = 0;

×

791

    taosArrayClear(array);

×

793

  code = blockDataUpdateTsWindow(pBlock, 0);

×

794

  QUERY_CHECK_CODE(code, lino, _end);

×

796

_end:

×

797

  if (code != TSDB_CODE_SUCCESS) {

×

798

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

800

×

802

static int32_t processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval,

×

804

  int32_t                      code = TSDB_CODE_SUCCESS;

×

805

  int32_t                      lino = 0;

×

806

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

807

  SColumnInfoData*             pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);

×

808

  TSKEY*                       tsData = (TSKEY*)pStartCol->pData;

×

809

  SColumnInfoData*             pEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);

×

810

  TSKEY*                       tsEndData = (TSKEY*)pEndCol->pData;

×

811

  SColumnInfoData*             pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);

×

812

  uint64_t*                    groupIdData = (uint64_t*)pGroupCol->pData;

×

813

  int32_t                      chId = getChildIndex(pBlock);

×

814

  bool                         res = false;

×

815

  for (int32_t i = 0; i < pBlock->info.rows; i++) {

×

816

    TSKEY winTs = tsData[i];

×

817

    while (winTs <= tsEndData[i]) {

×

818

      SWinKey winRes = {.ts = winTs, .groupId = groupIdData[i]};

×

819

      void*   chIds = taosHashGet(pMap, &winRes, sizeof(SWinKey));

×

820

      if (chIds) {

×

821

        SArray* chArray = *(SArray**)chIds;

×

822

        int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);

×

823

        if (index != -1) {

×

824

          qDebug("===stream===retrive window %" PRId64 " delete child id %d", winRes.ts, chId);

×

825

          taosArrayRemove(chArray, index);

×

826

          if (taosArrayGetSize(chArray) == 0) {

×

828

            taosArrayDestroy(chArray);

×

829

            int32_t tmpRes = taosHashRemove(pMap, &winRes, sizeof(SWinKey));

×

830

            qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

831

            res = true;

×

832

            qDebug("===stream===retrive pull data over.window %" PRId64, winRes.ts);

×

834

            void* pFinalCh = taosHashGet(pFinalMap, &winRes, sizeof(SWinKey));

×

835

            if (pFinalCh) {

×

836

              int32_t tmpRes = taosHashRemove(pFinalMap, &winRes, sizeof(SWinKey));

×

837

              qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

838

              doDeleteWindow(pOperator, winRes.ts, winRes.groupId);

×

839

              STimeWindow     nextWin = getFinalTimeWindow(winRes.ts, pInterval);

×

840

              SPullWindowInfo pull = {.window = nextWin,

×

841

                                      .groupId = winRes.groupId,

×

842

                                      .calWin.skey = nextWin.skey,

×

843

                                      .calWin.ekey = nextWin.skey};

×

845

              qDebug("===stream===prepare final retrive for delete window:%" PRId64 ",groupId:%" PRId64 ", size:%d",

×

847

              if (IS_MID_INTERVAL_OP(pOperator)) {

×

848

                SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info;

×

850

                void* tmp = taosArrayPush(pInfo->pMidPullDatas, &winRes);

×

851

                if (!tmp) {

×

852

                  code = terrno;

×

853

                  QUERY_CHECK_CODE(code, lino, _end);

×

855

              } else if (savePullWindow(&pull, pPullWins) == TSDB_CODE_SUCCESS) {

×

856

                void* tmp = taosArrayPush(pInfo->pDelWins, &winRes);

×

857

                if (!tmp) {

×

858

                  code = terrno;

×

859

                  QUERY_CHECK_CODE(code, lino, _end);

×

862

                code = addPullWindow(pMap, &winRes, numOfCh);

×

863

                QUERY_CHECK_CODE(code, lino, _end);

×

865

                if (pInfo->destHasPrimaryKey) {

×

866

                  code = tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0);

×

867

                  QUERY_CHECK_CODE(code, lino, _end);

×

869

                qDebug("===stream===prepare final retrive for delete %" PRId64 ", size:%d", winRes.ts, numOfCh);

×

875

      winTs = taosTimeAdd(winTs, pInterval->sliding, pInterval->slidingUnit, pInterval->precision, NULL);

×

878

  if (pBeOver) {

×

879

    *pBeOver = res;

×

882

_end:

×

883

  if (code != TSDB_CODE_SUCCESS) {

×

884

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

886

  return code;

×

889

static int32_t addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, int32_t childId) {

×

890

  int32_t code = TSDB_CODE_SUCCESS;

×

891

  int32_t lino = 0;

×

892

  int32_t size = taosArrayGetSize(wins);

×

893

  for (int32_t i = 0; i < size; i++) {

×

894

    SWinKey*    winKey = taosArrayGet(wins, i);

×

895

    STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval);

×

896

    void*       chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey));

×

897

    if (!chIds) {

×

898

      SPullWindowInfo pull = {

×

899

          .window = nextWin, .groupId = winKey->groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey};

×

901

      if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) {

×

902

        code = addPullWindow(pInfo->pPullDataMap, winKey, pInfo->numOfChild);

×

903

        QUERY_CHECK_CODE(code, lino, _end);

×

905

        if (pInfo->destHasPrimaryKey) {

×

906

          code = tSimpleHashPut(pInfo->pDeletedMap, winKey, sizeof(SWinKey), NULL, 0);

×

907

          QUERY_CHECK_CODE(code, lino, _end);

×

909

        qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, pInfo->numOfChild);

×

912

      SArray* chArray = *(void**)chIds;

×

913

      int32_t index = taosArraySearchIdx(chArray, &childId, compareInt32Val, TD_EQ);

×

914

      qDebug("===stream===check final retrive %" PRId64 ",chid:%d", winKey->ts, index);

×

915

      if (index == -1) {

×

916

        qDebug("===stream===add final retrive %" PRId64, winKey->ts);

×

917

        code = taosHashPut(pInfo->pFinalPullDataMap, winKey, sizeof(SWinKey), NULL, 0);

×

918

        QUERY_CHECK_CODE(code, lino, _end);

×

923

_end:

×

924

  if (code != TSDB_CODE_SUCCESS) {

×

925

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

927

  return code;

×

930

static void clearFunctionContext(SExprSupp* pSup) {

×

931

  for (int32_t i = 0; i < pSup->numOfExprs; i++) {

×

932

    pSup->pCtx[i].saveHandle.currentPage = -1;

×

934

×

936

int32_t getOutputBuf(void* pState, SRowBuffPos* pPos, SResultRow** pResult, SStateStore* pStore) {

×

937

  return pStore->streamStateGetByPos(pState, pPos, (void**)pResult);

×

940

void buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup,

×

942

  int32_t         code = TSDB_CODE_SUCCESS;

×

943

  int32_t         lino = 0;

×

944

  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;

×

945

  SStorageAPI*    pAPI = &pOperator->pTaskInfo->storageAPI;

×

946

  SExprInfo*      pExprInfo = pSup->pExprInfo;

×

947

  int32_t         numOfExprs = pSup->numOfExprs;

×

948

  int32_t*        rowEntryOffset = pSup->rowEntryInfoOffset;

×

949

  SqlFunctionCtx* pCtx = pSup->pCtx;

×

951

  int32_t numOfRows = getNumOfTotalRes(pGroupResInfo);

×

953

  for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) {

×

954

    SRowBuffPos* pPos = *(SRowBuffPos**)taosArrayGet(pGroupResInfo->pRows, i);

×

955

    SResultRow*  pRow = NULL;

×

956

    code = getOutputBuf(pState, pPos, &pRow, &pAPI->stateStore);

×

957

    QUERY_CHECK_CODE(code, lino, _end);

×

958

    uint64_t groupId = ((SWinKey*)pPos->pKey)->groupId;

×

959

    doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset);

×

961

    if (pRow->numOfRows == 0) {

×

962

      pGroupResInfo->index += 1;

×

963

      continue;

×

965

    if (pBlock->info.id.groupId == 0) {

×

966

      pBlock->info.id.groupId = groupId;

×

967

      void*   tbname = NULL;

×

968

      int32_t winCode = TSDB_CODE_SUCCESS;

×

969

      code = pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname,

×

971

      QUERY_CHECK_CODE(code, lino, _end);

×

972

      if (winCode != TSDB_CODE_SUCCESS) {

×

973

        pBlock->info.parTbName[0] = 0;

×

975

        memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);

×

977

      pAPI->stateStore.streamStateFreeVal(tbname);

×

980

      if (pBlock->info.id.groupId != groupId) {

×

981

        break;

×

985

    if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) {

×

986

      break;

×

988

    pGroupResInfo->index += 1;

×

990

    for (int32_t j = 0; j < numOfExprs; ++j) {

×

991

      int32_t slotId = pExprInfo[j].base.resSchema.slotId;

×

993

      pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset);

×

995

      if (pCtx[j].fpSet.finalize) {

×

996

        int32_t tmpRes = pCtx[j].fpSet.finalize(&pCtx[j], pBlock);

×

997

        if (TAOS_FAILED(tmpRes)) {

×

998

          qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(tmpRes));

×

999

          QUERY_CHECK_CODE(code, lino, _end);

×

1001

      } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) {

×

1006

        SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId);

×

1007

        char*            in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo);

×

1008

        for (int32_t k = 0; k < pRow->numOfRows; ++k) {

×

1009

          code = colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes);

×

1010

          QUERY_CHECK_CODE(code, lino, _end);

×

1015

    if (pSessionKeys) {

×

1016

      SSessionKey key = {.groupId = groupId, .win = pRow->win};

×

1017

      for (int32_t j = 0; j < pRow->numOfRows; ++j) {

×

1018

        const void* px = taosArrayPush(pSessionKeys, &key);

×

1019

        QUERY_CHECK_NULL(px, code, lino, _end, terrno);

×

1023

    pBlock->info.rows += pRow->numOfRows;

×

1026

  pBlock->info.dataLoad = 1;

×

1027

  code = blockDataUpdateTsWindow(pBlock, 0);

×

1028

  QUERY_CHECK_CODE(code, lino, _end);

×

1030

_end:

×

1031

  if (code != TSDB_CODE_SUCCESS) {

×

1032

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

1034

×

1036

void doBuildStreamIntervalResult(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock,

×

1038

  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

×

1040

  pBlock->info.version = pTaskInfo->version;

×

1042

  blockDataCleanup(pBlock);

×

1043

  taosArrayClear(pSessionKeys);

×

1044

  if (!hasRemainResults(pGroupResInfo)) {

×

1045

    return;

×

1049

  pBlock->info.id.groupId = 0;

×

1050

  buildDataBlockFromGroupRes(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo, pSessionKeys);

×

1053

static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,

×

1055

  int32_t startPos = prevPosition + 1;

×

1056

  if (startPos == pDataBlockInfo->rows) {

×

1057

    startPos = -1;

×

1059

    *pNext = getFinalTimeWindow(primaryKeys[startPos], pInterval);

×

1061

  return startPos;

×

1064

static int32_t doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, uint64_t groupId,

×

1066

  int32_t                      code = TSDB_CODE_SUCCESS;

×

1067

  int32_t                      lino = 0;

×

1068

  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info;

×

1069

  pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);

×

1071

  SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo);

×

1072

  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;

×

1073

  SExprSupp*      pSup = &pOperator->exprSupp;

×

1074

  int32_t         numOfOutput = pSup->numOfExprs;

×

1075

  int32_t         step = 1;

×

1076

  TSKEY*          tsCols = NULL;

×

1077

  SRowBuffPos*    pResPos = NULL;

×

1078

  SResultRow*     pResult = NULL;

×

1079

  int32_t         forwardRows = 0;

×

1080

  int32_t         endRowId = pSDataBlock->info.rows - 1;

×

1082

  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);

×

1083

  tsCols = (int64_t*)pColDataInfo->pData;

×

1085

  void*            pPkVal = NULL;

×

1086

  int32_t          pkLen = 0;

×

1087

  SColumnInfoData* pPkColDataInfo = NULL;

×

1088

  if (hasSrcPrimaryKeyCol(&pInfo->basic)) {

×

1089

    pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex);

×

1092

  if (pSDataBlock->info.window.skey != tsCols[0] || pSDataBlock->info.window.ekey != tsCols[endRowId]) {

×

1093

    qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 ",maxKey %" PRId64,

×

1095

    code = blockDataUpdateTsWindow(pSDataBlock, pInfo->primaryTsIndex);

×

1096

    QUERY_CHECK_CODE(code, lino, _end);

×

1099

    if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) {

×

1100

      qError("table uid %" PRIu64 " data block timestamp is out of range! minKey %" PRId64 ",maxKey %" PRId64,

×

1105

  int32_t     startPos = 0;

×

1106

  TSKEY       ts = getStartTsKey(&pSDataBlock->info.window, tsCols);

×

1107

  STimeWindow nextWin = {0};

×

1108

  if (IS_FINAL_INTERVAL_OP(pOperator)) {

×

1109

    nextWin = getFinalTimeWindow(ts, &pInfo->interval);

×

1111

    nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC);

×

1113

  while (1) {

×

1114

    bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);

×

1115

    if (hasSrcPrimaryKeyCol(&pInfo->basic) && !IS_FINAL_INTERVAL_OP(pOperator) && pInfo->ignoreExpiredData &&

×

1116

        pSDataBlock->info.type != STREAM_PULL_DATA) {

×

1117

      pPkVal = colDataGetData(pPkColDataInfo, startPos);

×

1118

      pkLen = colDataGetRowLength(pPkColDataInfo, startPos);

×

1121

    if ((!IS_FINAL_INTERVAL_OP(pOperator) && pInfo->ignoreExpiredData && pSDataBlock->info.type != STREAM_PULL_DATA &&

×

1122

         checkExpiredData(&pInfo->stateStore, pInfo->pUpdateInfo, &pInfo->twAggSup, pSDataBlock->info.id.uid,

×

1123

                          nextWin.ekey, pPkVal, pkLen)) ||

×

1124

        !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) {

×

1125

      startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);

×

1126

      if (startPos < 0) {

×

1127

        break;

×

1129

      qDebug("===stream===ignore expired data, window end ts:%" PRId64 ", maxts - wartermak:%" PRId64, nextWin.ekey,

×

1131

      continue;

×

1134

    if (IS_FINAL_INTERVAL_OP(pOperator) && pInfo->numOfChild > 0) {

×

1135

      bool    ignore = true;

×

1136

      SWinKey winRes = {

×

1137

          .ts = nextWin.skey,

×

1140

      void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey));

×

1141

      if (isDeletedStreamWindow(&nextWin, groupId, pInfo->pState, &pInfo->twAggSup, &pInfo->stateStore) && isClosed &&

×

1142

          !chIds) {

×

1143

        SPullWindowInfo pull = {

×

1144

            .window = nextWin, .groupId = groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey};

×

1146

        if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) {

×

1147

          code = addPullWindow(pInfo->pPullDataMap, &winRes, pInfo->numOfChild);

×

1148

          QUERY_CHECK_CODE(code, lino, _end);

×

1150

          if (pInfo->destHasPrimaryKey) {

×

1151

            code = tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0);

×

1152

            QUERY_CHECK_CODE(code, lino, _end);

×

1156

        int32_t index = -1;

×

1157

        SArray* chArray = NULL;

×

1158

        int32_t chId = 0;

×

1159

        if (chIds) {

×

1160

          chArray = *(void**)chIds;

×

1161

          chId = getChildIndex(pSDataBlock);

×

1162

          index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);

×

1164

        if (index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA) {

×

1165

          ignore = false;

×

1169

      if (ignore) {

×

1170

        startPos = getNextQualifiedFinalWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, startPos);

×

1171

        if (startPos < 0) {

×

1172

          break;

×

1174

        continue;

×

1178

    int32_t winCode = TSDB_CODE_SUCCESS;

×

1179

    code = setIntervalOutputBuf(pInfo->pState, &nextWin, &pResPos, groupId, pSup->pCtx, numOfOutput,

×

1181

    QUERY_CHECK_CODE(code, lino, _end);

×

1183

    if (winCode != TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator) &&

×

1184

        BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {

×

1185

      SSessionKey key = {.win = nextWin, .groupId = groupId};

×

1186

      code = addIntervalAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &key, &pInfo->basic.notifyEventSup,

×

1188

      QUERY_CHECK_CODE(code, lino, _end);

×

1191

    pResult = (SResultRow*)pResPos->pRowBuff;

×

1193

    if (IS_FINAL_INTERVAL_OP(pOperator)) {

×

1194

      forwardRows = 1;

×

1196

      forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey,

×

1200

    SWinKey key = {

×

1201

        .ts = pResult->win.skey,

×

1205

    if (pInfo->destHasPrimaryKey && winCode == TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator)) {

×

1206

      code = tSimpleHashPut(pDeletedMap, &key, sizeof(SWinKey), NULL, 0);

×

1207

      QUERY_CHECK_CODE(code, lino, _end);

×

1210

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) {

×

1211

      code = saveWinResult(&key, pResPos, pUpdatedMap);

×

1212

      QUERY_CHECK_CODE(code, lino, _end);

×

1215

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

1216

      pResPos->beUpdated = true;

×

1217

      code = tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pResPos, POINTER_BYTES);

×

1218

      QUERY_CHECK_CODE(code, lino, _end);

×

1221

    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, 1);

×

1222

    code = applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos,

×

1223

                                           forwardRows, pSDataBlock->info.rows, numOfOutput);

×

1224

    QUERY_CHECK_CODE(code, lino, _end);

×

1225

    key.ts = nextWin.skey;

×

1227

    if (pInfo->delKey.ts > key.ts) {

×

1228

      pInfo->delKey = key;

×

1230

    int32_t prevEndPos = (forwardRows - 1) * step + startPos;

×

1231

    if (IS_FINAL_INTERVAL_OP(pOperator)) {

×

1232

      startPos = getNextQualifiedFinalWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos);

×

1235

          getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC);

×

1237

    if (startPos < 0) {

×

1238

      break;

×

1241

_end:

×

1242

  if (code != TSDB_CODE_SUCCESS) {

×

1243

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

1245

  return code;

×

1248

int winPosCmprImpl(const void* pKey1, const void* pKey2) {

×

1249

  SRowBuffPos* pos1 = *(SRowBuffPos**)pKey1;

×

1250

  SRowBuffPos* pos2 = *(SRowBuffPos**)pKey2;

×

1251

  SWinKey*     pWin1 = (SWinKey*)pos1->pKey;

×

1252

  SWinKey*     pWin2 = (SWinKey*)pos2->pKey;

×

1254

  if (pWin1->groupId > pWin2->groupId) {

×

1255

    return 1;

×

1256

  } else if (pWin1->groupId < pWin2->groupId) {

×

1257

    return -1;

×

1260

  if (pWin1->ts > pWin2->ts) {

×

1261

    return 1;

×

1262

  } else if (pWin1->ts < pWin2->ts) {

×

1263

    return -1;

×

1266

  return 0;

×

1269

static void resetUnCloseWinInfo(SSHashObj* winMap) {

×

1270

  void*   pIte = NULL;

×

1271

  int32_t iter = 0;

×

1272

  while ((pIte = tSimpleHashIterate(winMap, pIte, &iter)) != NULL) {

×

1273

    SRowBuffPos* pPos = *(SRowBuffPos**)pIte;

×

1274

    pPos->beUsed = true;

×

1276

×

1278

int32_t encodeSWinKey(void** buf, SWinKey* key) {

×

1279

  int32_t tlen = 0;

×

1280

  tlen += taosEncodeFixedI64(buf, key->ts);

×

1281

  tlen += taosEncodeFixedU64(buf, key->groupId);

×

1282

  return tlen;

×

1285

void* decodeSWinKey(void* buf, SWinKey* key) {

×

1286

  buf = taosDecodeFixedI64(buf, &key->ts);

×

1287

  buf = taosDecodeFixedU64(buf, &key->groupId);

×

1288

  return buf;

×

1291

int32_t encodeSTimeWindowAggSupp(void** buf, STimeWindowAggSupp* pTwAggSup) {

×

1292

  int32_t tlen = 0;

×

1293

  tlen += taosEncodeFixedI64(buf, pTwAggSup->minTs);

×

1294

  tlen += taosEncodeFixedI64(buf, pTwAggSup->maxTs);

×

1295

  return tlen;

×

1298

void* decodeSTimeWindowAggSupp(void* buf, STimeWindowAggSupp* pTwAggSup) {

×

1299

  buf = taosDecodeFixedI64(buf, &pTwAggSup->minTs);

×

1300

  buf = taosDecodeFixedI64(buf, &pTwAggSup->maxTs);

×

1301

  return buf;

×

1304

int32_t encodeSTimeWindow(void** buf, STimeWindow* pWin) {

×

1305

  int32_t tlen = 0;

×

1306

  tlen += taosEncodeFixedI64(buf, pWin->skey);

×

1307

  tlen += taosEncodeFixedI64(buf, pWin->ekey);

×

1308

  return tlen;

×

1311

void* decodeSTimeWindow(void* buf, STimeWindow* pWin) {

×

1312

  buf = taosDecodeFixedI64(buf, &pWin->skey);

×

1313

  buf = taosDecodeFixedI64(buf, &pWin->ekey);

×

1314

  return buf;

×

1317

int32_t encodeSPullWindowInfo(void** buf, SPullWindowInfo* pPullInfo) {

×

1318

  int32_t tlen = 0;

×

1319

  tlen += encodeSTimeWindow(buf, &pPullInfo->calWin);

×

1320

  tlen += taosEncodeFixedU64(buf, pPullInfo->groupId);

×

1321

  tlen += encodeSTimeWindow(buf, &pPullInfo->window);

×

1322

  return tlen;

×

1325

void* decodeSPullWindowInfo(void* buf, SPullWindowInfo* pPullInfo) {

×

1326

  buf = decodeSTimeWindow(buf, &pPullInfo->calWin);

×

1327

  buf = taosDecodeFixedU64(buf, &pPullInfo->groupId);

×

1328

  buf = decodeSTimeWindow(buf, &pPullInfo->window);

×

1329

  return buf;

×

1332

int32_t encodeSPullWindowInfoArray(void** buf, SArray* pPullInfos) {

×

1333

  int32_t tlen = 0;

×

1334

  int32_t size = taosArrayGetSize(pPullInfos);

×

1335

  tlen += taosEncodeFixedI32(buf, size);

×

1336

  for (int32_t i = 0; i < size; i++) {

×

1337

    void* pItem = taosArrayGet(pPullInfos, i);

×

1338

    tlen += encodeSPullWindowInfo(buf, pItem);

×

1340

  return tlen;

×

1343

int32_t decodeSPullWindowInfoArray(void* buf, SArray* pPullInfos, void** ppBuf) {

×

1344

  int32_t code = TSDB_CODE_SUCCESS;

×

1345

  int32_t lino = 0;

×

1346

  int32_t size = 0;

×

1347

  buf = taosDecodeFixedI32(buf, &size);

×

1348

  for (int32_t i = 0; i < size; i++) {

×

1349

    SPullWindowInfo item = {0};

×

1350

    buf = decodeSPullWindowInfo(buf, &item);

×

1351

    void* tmp = taosArrayPush(pPullInfos, &item);

×

1352

    if (!tmp) {

×

1353

      code = terrno;

×

1354

      QUERY_CHECK_CODE(code, lino, _end);

×

1357

  (*ppBuf) = buf;

×

1359

_end:

×

1360

  if (code != TSDB_CODE_SUCCESS) {

×

1361

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

1363

  return code;

×

1366

int32_t doStreamIntervalEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOperator) {

×

1367

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1368

  if (!pInfo) {

×

1369

    return 0;

×

1372

  void* pData = (buf == NULL) ? NULL : *buf;

×

1375

  int32_t tlen = 0;

×

1376

  int32_t mapSize = tSimpleHashGetSize(pInfo->aggSup.pResultRowHashTable);

×

1377

  tlen += taosEncodeFixedI32(buf, mapSize);

×

1378

  void*   pIte = NULL;

×

1379

  size_t  keyLen = 0;

×

1380

  int32_t iter = 0;

×

1381

  while ((pIte = tSimpleHashIterate(pInfo->aggSup.pResultRowHashTable, pIte, &iter)) != NULL) {

×

1382

    void* key = tSimpleHashGetKey(pIte, &keyLen);

×

1383

    tlen += encodeSWinKey(buf, key);

×

1387

  tlen += encodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);

×

1390

  int32_t size = taosHashGetSize(pInfo->pPullDataMap);

×

1391

  tlen += taosEncodeFixedI32(buf, size);

×

1392

  pIte = NULL;

×

1393

  keyLen = 0;

×

1394

  while ((pIte = taosHashIterate(pInfo->pPullDataMap, pIte)) != NULL) {

×

1395

    void* key = taosHashGetKey(pIte, &keyLen);

×

1396

    tlen += encodeSWinKey(buf, key);

×

1397

    SArray* pArray = *(SArray**)pIte;

×

1398

    int32_t chSize = taosArrayGetSize(pArray);

×

1399

    tlen += taosEncodeFixedI32(buf, chSize);

×

1400

    for (int32_t i = 0; i < chSize; i++) {

×

1401

      void* pChItem = taosArrayGet(pArray, i);

×

1402

      tlen += taosEncodeFixedI32(buf, *(int32_t*)pChItem);

×

1407

  tlen += encodeSPullWindowInfoArray(buf, pInfo->pPullWins);

×

1410

  tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);

×

1413

  tlen += encodeStreamBasicInfo(buf, &pInfo->basic);

×

1416

  if (buf) {

×

1417

    uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));

×

1418

    tlen += taosEncodeFixedU32(buf, cksum);

×

1420

    tlen += sizeof(uint32_t);

×

1423

  return tlen;

×

1426

void doStreamIntervalDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator) {

×

1427

  int32_t                      code = TSDB_CODE_SUCCESS;

×

1428

  int32_t                      lino = 0;

×

1429

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1430

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

1431

  void*                        pDataEnd = POINTER_SHIFT(buf, len);

×

1432

  if (!pInfo) {

×

1433

    code = TSDB_CODE_FAILED;

×

1434

    QUERY_CHECK_CODE(code, lino, _end);

×

1438

  int32_t dataLen = len - sizeof(uint32_t);

×

1439

  void*   pCksum = POINTER_SHIFT(buf, dataLen);

×

1440

  if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) {

×

1441

    code = TSDB_CODE_FAILED;

×

1442

    QUERY_CHECK_CODE(code, lino, _end);

×

1444

  pDataEnd = pCksum;

×

1447

  int32_t mapSize = 0;

×

1448

  buf = taosDecodeFixedI32(buf, &mapSize);

×

1449

  for (int32_t i = 0; i < mapSize; i++) {

×

1450

    SWinKey key = {0};

×

1451

    buf = decodeSWinKey(buf, &key);

×

1452

    SRowBuffPos* pPos = NULL;

×

1453

    int32_t      resSize = pInfo->aggSup.resultRowSize;

×

1454

    int32_t      winCode = TSDB_CODE_SUCCESS;

×

1455

    code = pInfo->stateStore.streamStateAddIfNotExist(pInfo->pState, &key, (void**)&pPos, &resSize, &winCode);

×

1456

    QUERY_CHECK_CODE(code, lino, _end);

×

1457

    QUERY_CHECK_CONDITION((winCode == TSDB_CODE_SUCCESS), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);

×

1459

    code = tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pPos, POINTER_BYTES);

×

1460

    QUERY_CHECK_CODE(code, lino, _end);

×

1464

  buf = decodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);

×

1467

  int32_t size = 0;

×

1468

  buf = taosDecodeFixedI32(buf, &size);

×

1469

  for (int32_t i = 0; i < size; i++) {

×

1470

    SWinKey key = {0};

×

1471

    SArray* pArray = taosArrayInit(0, sizeof(int32_t));

×

1472

    if (!pArray) {

×

1473

      code = terrno;

×

1474

      QUERY_CHECK_CODE(code, lino, _end);

×

1477

    buf = decodeSWinKey(buf, &key);

×

1478

    int32_t chSize = 0;

×

1479

    buf = taosDecodeFixedI32(buf, &chSize);

×

1480

    for (int32_t i = 0; i < chSize; i++) {

×

1481

      int32_t chId = 0;

×

1482

      buf = taosDecodeFixedI32(buf, &chId);

×

1483

      void* tmp = taosArrayPush(pArray, &chId);

×

1484

      if (!tmp) {

×

1485

        code = terrno;

×

1486

        QUERY_CHECK_CODE(code, lino, _end);

×

1489

    code = taosHashPut(pInfo->pPullDataMap, &key, sizeof(SWinKey), &pArray, POINTER_BYTES);

×

1490

    QUERY_CHECK_CODE(code, lino, _end);

×

1494

  code = decodeSPullWindowInfoArray(buf, pInfo->pPullWins, &buf);

×

1495

  QUERY_CHECK_CODE(code, lino, _end);

×

1498

  buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);

×

1501

  if (buf < pDataEnd) {

×

1502

    code = decodeStreamBasicInfo(&buf, &pInfo->basic);

×

1503

    QUERY_CHECK_CODE(code, lino, _end);

×

1506

_end:

×

1507

  if (code != TSDB_CODE_SUCCESS) {

×

1508

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

1510

×

1512

void doStreamIntervalSaveCheckpoint(SOperatorInfo* pOperator) {

×

1513

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1514

  if (needSaveStreamOperatorInfo(&pInfo->basic)) {

×

1515

    int32_t len = doStreamIntervalEncodeOpState(NULL, 0, pOperator);

×

1516

    void*   buf = taosMemoryCalloc(1, len);

×

1517

    if (!buf) {

×

1518

      qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno));

×

1519

      return;

×

1521

    void* pBuf = buf;

×

1522

    len = doStreamIntervalEncodeOpState(&pBuf, len, pOperator);

×

1523

    pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_INTERVAL_OP_CHECKPOINT_NAME,

×

1525

    taosMemoryFree(buf);

×

1526

    saveStreamOperatorStateComplete(&pInfo->basic);

×

1530

int32_t copyIntervalDeleteKey(SSHashObj* pMap, SArray* pWins) {

×

1531

  int32_t code = TSDB_CODE_SUCCESS;

×

1532

  int32_t lino = 0;

×

1533

  void*   pIte = NULL;

×

1534

  int32_t iter = 0;

×

1535

  while ((pIte = tSimpleHashIterate(pMap, pIte, &iter)) != NULL) {

×

1536

    void* pKey = tSimpleHashGetKey(pIte, NULL);

×

1537

    void* tmp = taosArrayPush(pWins, pKey);

×

1538

    if (!tmp) {

×

1539

      code = terrno;

×

1540

      QUERY_CHECK_CODE(code, lino, _end);

×

1543

  tSimpleHashClear(pMap);

×

1545

_end:

×

1546

  if (code != TSDB_CODE_SUCCESS) {

×

1547

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

1549

  return code;

×

1552

static int32_t buildIntervalResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

1553

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1554

  int32_t                      code = TSDB_CODE_SUCCESS;

×

1555

  int32_t                      lino = 0;

×

1556

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

1557

  uint16_t                     opType = pOperator->operatorType;

×

1558

  SStreamNotifyEventSupp*      pNotifySup = &pInfo->basic.notifyEventSup;

×

1559

  STaskNotifyEventStat*        pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;

×

1560

  bool                         addNotifyEvent = false;

×

1563

  if (isTaskKilled(pTaskInfo)) {

×

1564

    (*ppRes) = NULL;

×

1565

    return code;

×

1568

  addNotifyEvent = IS_NORMAL_INTERVAL_OP(pOperator) &&

×

1569

                   BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);

×

1570

  if (IS_FINAL_INTERVAL_OP(pOperator)) {

×

1571

    doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);

×

1572

    if (pInfo->pPullDataRes->info.rows != 0) {

×

1574

      printDataBlock(pInfo->pPullDataRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));

×

1575

      (*ppRes) = pInfo->pPullDataRes;

×

1576

      return code;

×

1580

  doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);

×

1581

  if (pInfo->pDelRes->info.rows != 0) {

×

1583

    printDataBlock(pInfo->pDelRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));

×

1584

    if (addNotifyEvent) {

×

1585

      code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);

×

1586

      QUERY_CHECK_CODE(code, lino, _end);

×

1588

    (*ppRes) = pInfo->pDelRes;

×

1589

    return code;

×

1592

  doBuildStreamIntervalResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo,

×

1594

  if (pInfo->binfo.pRes->info.rows != 0) {

×

1595

    printDataBlock(pInfo->binfo.pRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));

×

1596

    if (addNotifyEvent) {

×

1597

      code = addAggResultNotifyEvent(pInfo->binfo.pRes, pNotifySup->pSessionKeys,

×

1598

                                     pTaskInfo->streamInfo.notifyResultSchema, pNotifySup, pNotifyEventStat);

×

1599

      QUERY_CHECK_CODE(code, lino, _end);

×

1601

    (*ppRes) = pInfo->binfo.pRes;

×

1602

    return code;

×

1605

  code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);

×

1606

  QUERY_CHECK_CODE(code, lino, _end);

×

1607

  if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {

×

1608

    printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

1609

    (*ppRes) = pNotifySup->pEventBlock;

×

1610

    return code;

×

1613

  code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);

×

1614

  QUERY_CHECK_CODE(code, lino, _end);

×

1616

_end:

×

1617

  if (code != TSDB_CODE_SUCCESS) {

×

1618

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

1620

  (*ppRes) = NULL;

×

1621

  return code;

×

1624

int32_t copyUpdateResult(SSHashObj** ppWinUpdated, SArray* pUpdated, __compar_fn_t compar) {

×

1625

  int32_t code = TSDB_CODE_SUCCESS;

×

1626

  int32_t lino = 0;

×

1627

  void*   pIte = NULL;

×

1628

  int32_t iter = 0;

×

1629

  while ((pIte = tSimpleHashIterate(*ppWinUpdated, pIte, &iter)) != NULL) {

×

1630

    void* tmp = taosArrayPush(pUpdated, pIte);

×

1631

    if (!tmp) {

×

1632

      code = terrno;

×

1633

      QUERY_CHECK_CODE(code, lino, _end);

×

1636

  taosArraySort(pUpdated, compar);

×

1637

  tSimpleHashCleanup(*ppWinUpdated);

×

1638

  *ppWinUpdated = NULL;

×

1640

_end:

×

1641

  if (code != TSDB_CODE_SUCCESS) {

×

1642

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

1644

  return code;

×

1647

static int32_t doStreamFinalIntervalAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

1648

  int32_t                      code = TSDB_CODE_SUCCESS;

×

1649

  int32_t                      lino = 0;

×

1650

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1651

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

1652

  SStorageAPI*                 pAPI = &pOperator->pTaskInfo->storageAPI;

×

1654

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

1655

  SExprSupp*     pSup = &pOperator->exprSupp;

×

1657

  qDebug("stask:%s  %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);

×

1659

  if (pOperator->status == OP_EXEC_DONE) {

×

1660

    (*ppRes) = NULL;

×

1661

    return code;

×

1662

  } else if (pOperator->status == OP_RES_TO_RETURN) {

×

1663

    SSDataBlock* resBlock = NULL;

×

1664

    code = buildIntervalResult(pOperator, &resBlock);

×

1665

    QUERY_CHECK_CODE(code, lino, _end);

×

1666

    if (resBlock != NULL) {

×

1667

      (*ppRes) = resBlock;

×

1668

      return code;

×

1671

    if (pInfo->recvGetAll) {

×

1672

      pInfo->recvGetAll = false;

×

1673

      resetUnCloseWinInfo(pInfo->aggSup.pResultRowHashTable);

×

1676

    if (pInfo->reCkBlock) {

×

1677

      pInfo->reCkBlock = false;

×

1678

      printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

1679

      (*ppRes) = pInfo->pCheckpointRes;

×

1680

      return code;

×

1683

    setStreamOperatorCompleted(pOperator);

×

1684

    if (!IS_FINAL_INTERVAL_OP(pOperator)) {

×

1685

      clearFunctionContext(&pOperator->exprSupp);

×

1687

      clearStreamIntervalOperator(pInfo);

×

1688

      qDebug("===stream===clear semi operator");

×

1690

    (*ppRes) = NULL;

×

1691

    return code;

×

1693

    if (!IS_FINAL_INTERVAL_OP(pOperator)) {

×

1694

      SSDataBlock* resBlock = NULL;

×

1695

      code = buildIntervalResult(pOperator, &resBlock);

×

1696

      QUERY_CHECK_CODE(code, lino, _end);

×

1697

      if (resBlock != NULL) {

×

1698

        (*ppRes) = resBlock;

×

1699

        return code;

×

1702

      if (pInfo->recvRetrive) {

×

1703

        pInfo->recvRetrive = false;

×

1704

        printDataBlock(pInfo->pMidRetriveRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

1705

        (*ppRes) = pInfo->pMidRetriveRes;

×

1706

        return code;

×

1711

  if (!pInfo->pUpdated) {

×

1712

    pInfo->pUpdated = taosArrayInit(4096, POINTER_BYTES);

×

1713

    QUERY_CHECK_NULL(pInfo->pUpdated, code, lino, _end, terrno);

×

1715

  if (!pInfo->pUpdatedMap) {

×

1716

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

1717

    pInfo->pUpdatedMap = tSimpleHashInit(4096, hashFn);

×

1718

    QUERY_CHECK_NULL(pInfo->pUpdatedMap, code, lino, _end, terrno);

×

1721

  while (1) {

×

1722

    if (isTaskKilled(pTaskInfo)) {

×

1723

      qInfo("===stream=== %s task is killed, code %s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code));

×

1724

      (*ppRes) = NULL;

×

1725

      return code;

×

1728

    SSDataBlock* pBlock = NULL;

×

1729

    code = downstream->fpSet.getNextFn(downstream, &pBlock);

×

1730

    QUERY_CHECK_CODE(code, lino, _end);

×

1732

    if (pBlock == NULL) {

×

1733

      pOperator->status = OP_RES_TO_RETURN;

×

1734

      qDebug("===stream===return data:%s. recv datablock num:%" PRIu64, getStreamOpName(pOperator->operatorType),

×

1736

      pInfo->numOfDatapack = 0;

×

1737

      break;

×

1740

    pInfo->numOfDatapack++;

×

1741

    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));

×

1742

    setStreamOperatorState(&pInfo->basic, pBlock->info.type);

×

1744

    if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) {

×

1745

      pInfo->binfo.pRes->info.type = pBlock->info.type;

×

1746

    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||

×

1747

               pBlock->info.type == STREAM_CLEAR) {

×

1748

      SArray* delWins = taosArrayInit(8, sizeof(SWinKey));

×

1749

      QUERY_CHECK_NULL(delWins, code, lino, _end, terrno);

×

1750

      SHashObj* finalMap = IS_FINAL_INTERVAL_OP(pOperator) ? pInfo->pFinalPullDataMap : NULL;

×

1751

      code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, finalMap);

×

1752

      QUERY_CHECK_CODE(code, lino, _end);

×

1754

      if (IS_FINAL_INTERVAL_OP(pOperator)) {

×

1755

        int32_t chId = getChildIndex(pBlock);

×

1756

        code = addRetriveWindow(delWins, pInfo, chId);

×

1757

        QUERY_CHECK_CODE(code, lino, _end);

×

1759

        if (pBlock->info.type != STREAM_CLEAR) {

×

1760

          void* tmp = taosArrayAddAll(pInfo->pDelWins, delWins);

×

1761

          if (!tmp && taosArrayGetSize(delWins) > 0) {

×

1762

            code = TSDB_CODE_OUT_OF_MEMORY;

×

1763

            QUERY_CHECK_CODE(code, lino, _end);

×

1766

        taosArrayDestroy(delWins);

×

1767

        continue;

×

1769

      removeResults(delWins, pInfo->pUpdatedMap);

×

1770

      void* tmp = taosArrayAddAll(pInfo->pDelWins, delWins);

×

1771

      if (!tmp && taosArrayGetSize(delWins) > 0) {

×

1772

        code = TSDB_CODE_OUT_OF_MEMORY;

×

1773

        QUERY_CHECK_CODE(code, lino, _end);

×

1775

      taosArrayDestroy(delWins);

×

1777

      doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);

×

1778

      if (pInfo->pDelRes->info.rows != 0) {

×

1780

        printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

1781

        if (pBlock->info.type == STREAM_CLEAR) {

×

1782

          pInfo->pDelRes->info.type = STREAM_CLEAR;

×

1784

          pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;

×

1786

        (*ppRes) = pInfo->pDelRes;

×

1787

        return code;

×

1790

      break;

×

1791

    } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_INTERVAL_OP(pOperator)) {

×

1792

      pInfo->recvGetAll = true;

×

1793

      code = getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap);

×

1794

      QUERY_CHECK_CODE(code, lino, _end);

×

1795

      continue;

×

1796

    } else if (pBlock->info.type == STREAM_RETRIEVE) {

×

1797

      if (!IS_FINAL_INTERVAL_OP(pOperator)) {

×

1798

        pInfo->recvRetrive = true;

×

1799

        code = copyDataBlock(pInfo->pMidRetriveRes, pBlock);

×

1800

        QUERY_CHECK_CODE(code, lino, _end);

×

1802

        pInfo->pMidRetriveRes->info.type = STREAM_MID_RETRIEVE;

×

1803

        code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, NULL, pInfo->pUpdatedMap, NULL);

×

1804

        QUERY_CHECK_CODE(code, lino, _end);

×

1805

        break;

×

1807

      continue;

×

1808

    } else if (pBlock->info.type == STREAM_PULL_OVER && IS_FINAL_INTERVAL_OP(pOperator)) {

×

1809

      code = processPullOver(pBlock, pInfo->pPullDataMap, pInfo->pFinalPullDataMap, &pInfo->interval, pInfo->pPullWins,

×

1811

      QUERY_CHECK_CODE(code, lino, _end);

×

1812

      continue;

×

1813

    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {

×

1814

      (*ppRes) = pBlock;

×

1815

      return code;

×

1816

    } else if (pBlock->info.type == STREAM_CHECKPOINT) {

×

1817

      pAPI->stateStore.streamStateCommit(pInfo->pState);

×

1818

      doStreamIntervalSaveCheckpoint(pOperator);

×

1819

      code = copyDataBlock(pInfo->pCheckpointRes, pBlock);

×

1820

      QUERY_CHECK_CODE(code, lino, _end);

×

1822

      continue;

×

1823

    } else if (IS_FINAL_INTERVAL_OP(pOperator) && pBlock->info.type == STREAM_MID_RETRIEVE) {

×

1824

      continue;

×

1826

      if (pBlock->info.type != STREAM_INVALID) {

×

1827

        code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;

×

1828

        QUERY_CHECK_CODE(code, lino, _end);

×

1832

    if (pInfo->scalarSupp.pExprInfo != NULL) {

×

1833

      SExprSupp* pExprSup = &pInfo->scalarSupp;

×

1834

      code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);

×

1835

      QUERY_CHECK_CODE(code, lino, _end);

×

1837

    code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

1838

    QUERY_CHECK_CODE(code, lino, _end);

×

1840

    code = doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pInfo->pUpdatedMap, pInfo->pDeletedMap);

×

1841

    if (code == TSDB_CODE_STREAM_INTERNAL_ERROR) {

×

1842

      code = TSDB_CODE_SUCCESS;

×

1843

      pOperator->status = OP_RES_TO_RETURN;

×

1844

      break;

×

1846

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);

×

1847

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.watermark);

×

1848

    pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey);

×

1851

  if (IS_FINAL_INTERVAL_OP(pOperator) && !pInfo->destHasPrimaryKey) {

×

1852

    removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins);

×

1854

  if (IS_FINAL_INTERVAL_OP(pOperator)) {

×

1855

    code = closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval,

×

1857

    QUERY_CHECK_CODE(code, lino, _end);

×

1859

    if (pInfo->destHasPrimaryKey) {

×

1860

      code = copyIntervalDeleteKey(pInfo->pDeletedMap, pInfo->pDelWins);

×

1861

      QUERY_CHECK_CODE(code, lino, _end);

×

1864

  pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs;

×

1866

  code = copyUpdateResult(&pInfo->pUpdatedMap, pInfo->pUpdated, winPosCmprImpl);

×

1867

  QUERY_CHECK_CODE(code, lino, _end);

×

1869

  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);

×

1870

  pInfo->pUpdated = NULL;

×

1871

  code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);

×

1872

  QUERY_CHECK_CODE(code, lino, _end);

×

1874

  SSDataBlock* resBlock = NULL;

×

1875

  code = buildIntervalResult(pOperator, &resBlock);

×

1876

  QUERY_CHECK_CODE(code, lino, _end);

×

1877

  if (resBlock != NULL) {

×

1878

    (*ppRes) = resBlock;

×

1879

    return code;

×

1882

  if (pInfo->recvRetrive) {

×

1883

    pInfo->recvRetrive = false;

×

1884

    printDataBlock(pInfo->pMidRetriveRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

1885

    (*ppRes) = pInfo->pMidRetriveRes;

×

1886

    return code;

×

1889

_end:

×

1890

  if (code != TSDB_CODE_SUCCESS) {

×

1891

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

1892

    pTaskInfo->code = code;

×

1893

    T_LONG_JMP(pTaskInfo->env, code);

×

1895

  setStreamOperatorCompleted(pOperator);

×

1896

  (*ppRes) = NULL;

×

1897

  return code;

×

1900

int64_t getDeleteMark(SWindowPhysiNode* pWinPhyNode, int64_t interval) {

×

1901

  if (pWinPhyNode->deleteMark <= 0) {

×

1902

    return DEAULT_DELETE_MARK;

×

1904

  int64_t deleteMark = TMAX(pWinPhyNode->deleteMark, pWinPhyNode->watermark);

×

1905

  deleteMark = TMAX(deleteMark, interval);

×

1906

  return deleteMark;

×

1909

int64_t getDeleteMarkFromOption(SStreamNodeOption* pOption) {

×

1910

  if (pOption->deleteMark <= 0) {

×

1911

    return DEAULT_DELETE_MARK;

×

1913

  int64_t deleteMark = TMAX(pOption->deleteMark, pOption->watermark);

×

1914

  return deleteMark;

×

1917

TSKEY compareTs(void* pKey) {

×

1918

  SWinKey* pWinKey = (SWinKey*)pKey;

×

1919

  return pWinKey->ts;

×

1922

static int32_t getSelectivityBufSize(SqlFunctionCtx* pCtx) {

×

1923

  if (pCtx->subsidiaries.rowLen == 0) {

×

1924

    int32_t rowLen = 0;

×

1925

    for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) {

×

1926

      SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j];

×

1927

      rowLen += pc->pExpr->base.resSchema.bytes;

×

1930

    return rowLen + pCtx->subsidiaries.num * sizeof(bool);

×

1932

    return pCtx->subsidiaries.rowLen;

×

1936

static int32_t getMaxFunResSize(SExprSupp* pSup, int32_t numOfCols) {

×

1937

  int32_t size = 0;

×

1938

  for (int32_t i = 0; i < numOfCols; ++i) {

×

1939

    int32_t resSize = getSelectivityBufSize(pSup->pCtx + i);

×

1940

    size = TMAX(size, resSize);

×

1942

  return size;

×

1945

static void streamIntervalReleaseState(SOperatorInfo* pOperator) {

×

1946

  if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL &&

×

1947

      pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {

×

1948

    SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1949

    int32_t                      resSize = sizeof(TSKEY);

×

1950

    pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME,

×

1951

                                          strlen(STREAM_INTERVAL_OP_STATE_NAME), &pInfo->twAggSup.maxTs, resSize);

×

1953

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1954

  SStorageAPI*                 pAPI = &pOperator->pTaskInfo->storageAPI;

×

1955

  pAPI->stateStore.streamStateCommit(pInfo->pState);

×

1956

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

1957

  if (downstream->fpSet.releaseStreamStateFn) {

×

1958

    downstream->fpSet.releaseStreamStateFn(downstream);

×

1960

×

1962

void streamIntervalReloadState(SOperatorInfo* pOperator) {

×

1963

  int32_t                      code = TSDB_CODE_SUCCESS;

×

1964

  int32_t                      lino = 0;

×

1965

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1966

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

1967

  if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL &&

×

1968

      pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {

×

1969

    int32_t size = 0;

×

1970

    void*   pBuf = NULL;

×

1971

    code = pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME,

×

1973

    QUERY_CHECK_CODE(code, lino, _end);

×

1975

    TSKEY ts = *(TSKEY*)pBuf;

×

1976

    taosMemoryFree(pBuf);

×

1977

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);

×

1978

    pInfo->stateStore.streamStateReloadInfo(pInfo->pState, ts);

×

1980

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

1981

  if (downstream->fpSet.reloadStreamStateFn) {

×

1982

    downstream->fpSet.reloadStreamStateFn(downstream);

×

1984

  reloadFromDownStream(downstream, pInfo);

×

1986

_end:

×

1987

  if (code != TSDB_CODE_SUCCESS) {

×

1988

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

1990

×

1992

int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,

×

1994

  QRY_PARAM_CHECK(pOptrInfo);

×

1996

  int32_t                      code = TSDB_CODE_SUCCESS;

×

1997

  int32_t                      lino = 0;

×

1998

  SIntervalPhysiNode*          pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;

×

1999

  SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo));

×

2000

  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));

×

2001

  if (pInfo == NULL || pOperator == NULL) {

×

2002

    code = terrno;

×

2003

    QUERY_CHECK_CODE(code, lino, _error);

×

2006

  pOperator->exprSupp.hasWindowOrGroup = true;

×

2007

  pOperator->pTaskInfo = pTaskInfo;

×

2008

  SStorageAPI* pAPI = &pTaskInfo->storageAPI;

×

2010

  pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval,

×

2011

                                .sliding = pIntervalPhyNode->sliding,

×

2012

                                .intervalUnit = pIntervalPhyNode->intervalUnit,

×

2013

                                .slidingUnit = pIntervalPhyNode->slidingUnit,

×

2014

                                .offset = pIntervalPhyNode->offset,

×

2015

                                .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision,

×

2016

                                .timeRange = pIntervalPhyNode->timeRange};

×

2017

  calcIntervalAutoOffset(&pInfo->interval);

×

2018

  pInfo->twAggSup = (STimeWindowAggSupp){

×

2019

      .waterMark = pIntervalPhyNode->window.watermark,

×

2020

      .calTrigger = pIntervalPhyNode->window.triggerType,

×

2023

      .deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval),

×

2027

  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;

×

2028

  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;

×

2029

  initResultSizeInfo(&pOperator->resultInfo, 4096);

×

2030

  if (pIntervalPhyNode->window.pExprs != NULL) {

×

2031

    int32_t    numOfScalar = 0;

×

2032

    SExprInfo* pScalarExprInfo = NULL;

×

2034

    code = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalar);

×

2035

    QUERY_CHECK_CODE(code, lino, _error);

×

2037

    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);

×

2038

    QUERY_CHECK_CODE(code, lino, _error);

×

2041

  SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);

×

2042

  QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);

×

2043

  initBasicInfo(&pInfo->binfo, pResBlock);

×

2045

  pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState));

×

2046

  QUERY_CHECK_NULL(pInfo->pState, code, lino, _error, terrno);

×

2047

  qInfo("open state %p", pInfo->pState);

×

2048

  pAPI->stateStore.streamStateCopyBackend(pTaskInfo->streamInfo.pState, pInfo->pState);

×

2051

  qInfo("copy state %p to %p", pTaskInfo->streamInfo.pState, pInfo->pState);

×

2053

  pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsIndex);

×

2055

  int32_t    numOfCols = 0;

×

2056

  SExprInfo* pExprInfo = NULL;

×

2057

  code = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);

×

2058

  QUERY_CHECK_CODE(code, lino, _error);

×

2060

  code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,

×

2061

                    pInfo->pState, &pTaskInfo->storageAPI.functionStore);

×

2062

  QUERY_CHECK_CODE(code, lino, _error);

×

2064

  code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

×

2065

  QUERY_CHECK_CODE(code, lino, _error);

×

2066

  initResultRowInfo(&pInfo->binfo.resultRowInfo);

×

2068

  pInfo->numOfChild = numOfChild;

×

2069

  pInfo->pPhyNode = NULL;

×

2070

  code = nodesCloneNode((SNode*)pPhyNode, (SNode**)&pInfo->pPhyNode);

×

2071

  if (TSDB_CODE_SUCCESS != code) {

×

2072

    goto _error;

×

2075

  pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo));

×

2076

  QUERY_CHECK_NULL(pInfo->pPullWins, code, lino, _error, terrno);

×

2077

  pInfo->pullIndex = 0;

×

2078

  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

2079

  pInfo->pPullDataMap = taosHashInit(64, hashFn, true, HASH_NO_LOCK);

×

2080

  pInfo->pFinalPullDataMap = taosHashInit(64, hashFn, true, HASH_NO_LOCK);

×

2082

  code = createSpecialDataBlock(STREAM_RETRIEVE, &pInfo->pPullDataRes);

×

2083

  QUERY_CHECK_CODE(code, lino, _error);

×

2085

  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;

×

2086

  pInfo->ignoreExpiredDataSaved = false;

×

2087

  code = createSpecialDataBlock(STREAM_DELETE_RESULT, &pInfo->pDelRes);

×

2088

  QUERY_CHECK_CODE(code, lino, _error);

×

2090

  pInfo->delIndex = 0;

×

2091

  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));

×

2092

  QUERY_CHECK_NULL(pInfo->pDelWins, code, lino, _error, terrno);

×

2093

  pInfo->delKey.ts = INT64_MAX;

×

2094

  pInfo->delKey.groupId = 0;

×

2095

  pInfo->numOfDatapack = 0;

×

2096

  pInfo->pUpdated = NULL;

×

2097

  pInfo->pUpdatedMap = NULL;

×

2098

  pInfo->stateStore = pTaskInfo->storageAPI.stateStore;

×

2099

  int32_t funResSize = getMaxFunResSize(&pOperator->exprSupp, numOfCols);

×

2100

  pInfo->pState->pFileState = NULL;

×

2103

  pInfo->pState->pResultRowStore.resultRowGet = getResultRowFromBuf;

×

2104

  pInfo->pState->pResultRowStore.resultRowPut = putResultRowToBuf;

×

2105

  pInfo->pState->pExprSupp = &pOperator->exprSupp;

×

2108

      pAPI->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize,

×

2109

                                           compareTs, pInfo->pState, pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo),

×

2110

                                           pHandle->checkpointId, STREAM_STATE_BUFF_HASH, &pInfo->pState->pFileState);

×

2111

  QUERY_CHECK_CODE(code, lino, _error);

×

2113

  pInfo->dataVersion = 0;

×

2114

  pInfo->recvGetAll = false;

×

2115

  pInfo->recvPullover = false;

×

2116

  pInfo->recvRetrive = false;

×

2118

  code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);

×

2119

  QUERY_CHECK_CODE(code, lino, _error);

×

2120

  code = createSpecialDataBlock(STREAM_MID_RETRIEVE, &pInfo->pMidRetriveRes);

×

2121

  QUERY_CHECK_CODE(code, lino, _error);

×

2122

  code = createSpecialDataBlock(STREAM_MID_RETRIEVE, &pInfo->pMidPulloverRes);

×

2123

  QUERY_CHECK_CODE(code, lino, _error);

×

2125

  pInfo->clearState = false;

×

2126

  pInfo->pMidPullDatas = taosArrayInit(4, sizeof(SWinKey));

×

2127

  QUERY_CHECK_NULL(pInfo->pMidPullDatas, code, lino, _error, terrno);

×

2128

  pInfo->pDeletedMap = tSimpleHashInit(4096, hashFn);

×

2129

  QUERY_CHECK_NULL(pInfo->pDeletedMap, code, lino, _error, terrno);

×

2130

  pInfo->destHasPrimaryKey = pIntervalPhyNode->window.destHasPrimaryKey;

×

2131

  pInfo->pOperator = pOperator;

×

2133

  pOperator->operatorType = pPhyNode->type;

×

2134

  if (!IS_FINAL_INTERVAL_OP(pOperator) || numOfChild == 0) {

×

2135

    pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;

×

2137

  pOperator->name = getStreamOpName(pOperator->operatorType);

×

2138

  pOperator->blocking = true;

×

2139

  pOperator->status = OP_NOT_OPENED;

×

2140

  pOperator->info = pInfo;

×

2142

  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {

×

2144

        createOperatorFpSet(NULL, doStreamMidIntervalAggNext, NULL, destroyStreamFinalIntervalOperatorInfo,

×

2148

        createOperatorFpSet(NULL, doStreamFinalIntervalAggNext, NULL, destroyStreamFinalIntervalOperatorInfo,

×

2151

  setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState);

×

2153

  code = initStreamBasicInfo(&pInfo->basic, pOperator);

×

2154

  QUERY_CHECK_CODE(code, lino, _error);

×

2156

  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL ||

×

2157

      pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {

×

2158

    pInfo->basic.primaryPkIndex = -1;

×

2159

    code = initIntervalDownStream(downstream, pPhyNode->type, pInfo, &pInfo->basic);

×

2160

    QUERY_CHECK_CODE(code, lino, _error);

×

2162

  code = appendDownstream(pOperator, &downstream, 1);

×

2163

  QUERY_CHECK_CODE(code, lino, _error);

×

2166

  void*   buff = NULL;

×

2167

  int32_t len = 0;

×

2168

  int32_t res = pAPI->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_CHECKPOINT_NAME,

×

2170

  if (res == TSDB_CODE_SUCCESS) {

×

2171

    doStreamIntervalDecodeOpState(buff, len, pOperator);

×

2172

    taosMemoryFree(buff);

×

2175

  *pOptrInfo = pOperator;

×

2176

  return TSDB_CODE_SUCCESS;

×

2178

_error:

×

2179

  if (pInfo != NULL) destroyStreamFinalIntervalOperatorInfo(pInfo);

×

2180

  destroyOperatorAndDownstreams(pOperator, &downstream, 1);

×

2181

  pTaskInfo->code = code;

×

2182

  return code;

×

2185

void destroyStreamAggSupporter(SStreamAggSupporter* pSup) {

×

2186

  tSimpleHashCleanup(pSup->pResultRows);

×

2187

  destroyDiskbasedBuf(pSup->pResultBuf);

×

2188

  blockDataDestroy(pSup->pScanBlock);

×

2189

  if (pSup->stateStore.streamFileStateDestroy != NULL) {

×

2190

    pSup->stateStore.streamFileStateDestroy(pSup->pState->pFileState);

×

2192

  taosMemoryFreeClear(pSup->pState);

×

2193

  taosMemoryFreeClear(pSup->pDummyCtx);

×

2194

×

2196

void destroyStreamSessionAggOperatorInfo(void* param) {

×

2197

  if (param == NULL) {

×

2198

    return;

×

2200

  SStreamSessionAggOperatorInfo* pInfo = (SStreamSessionAggOperatorInfo*)param;

×

2201

  cleanupBasicInfo(&pInfo->binfo);

×

2202

  if (pInfo->pOperator) {

×

2203

    cleanupResultInfoInStream(pInfo->pOperator->pTaskInfo, pInfo->streamAggSup.pState, &pInfo->pOperator->exprSupp,

×

2205

    pInfo->pOperator = NULL;

×

2208

  destroyStreamBasicInfo(&pInfo->basic);

×

2209

  cleanupExprSupp(&pInfo->scalarSupp);

×

2210

  clearGroupResInfo(&pInfo->groupResInfo);

×

2211

  taosArrayDestroyP(pInfo->pUpdated, destroyFlusedPos);

×

2212

  pInfo->pUpdated = NULL;

×

2213

  destroyStreamAggSupporter(&pInfo->streamAggSup);

×

2215

  if (pInfo->pChildren != NULL) {

×

2216

    int32_t size = taosArrayGetSize(pInfo->pChildren);

×

2217

    for (int32_t i = 0; i < size; i++) {

×

2218

      SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, i);

×

2219

      destroyOperator(pChild);

×

2221

    taosArrayDestroy(pInfo->pChildren);

×

2224

  colDataDestroy(&pInfo->twAggSup.timeWindowData);

×

2225

  blockDataDestroy(pInfo->pDelRes);

×

2226

  blockDataDestroy(pInfo->pWinBlock);

×

2227

  tSimpleHashCleanup(pInfo->pStUpdated);

×

2228

  tSimpleHashCleanup(pInfo->pStDeleted);

×

2229

  cleanupGroupResInfo(&pInfo->groupResInfo);

×

2231

  taosArrayDestroy(pInfo->historyWins);

×

2232

  blockDataDestroy(pInfo->pCheckpointRes);

×

2233

  tSimpleHashCleanup(pInfo->pPkDeleted);

×

2235

  taosMemoryFreeClear(param);

×

2238

int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfCols,

×

2240

  initBasicInfo(pBasicInfo, pResultBlock);

×

2241

  int32_t code = initExprSupp(pSup, pExprInfo, numOfCols, pStore);

×

2242

  if (code != TSDB_CODE_SUCCESS) {

×

2243

    return code;

×

2246

  for (int32_t i = 0; i < numOfCols; ++i) {

×

2247

    pSup->pCtx[i].saveHandle.pBuf = NULL;

×

2250

  return TSDB_CODE_SUCCESS;

×

2253

void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t nums) {

×

2254

  for (int i = 0; i < nums; i++) {

×

2255

    pDummy[i].functionId = pCtx[i].functionId;

×

2256

    pDummy[i].isNotNullFunc = pCtx[i].isNotNullFunc;

×

2257

    pDummy[i].isPseudoFunc = pCtx[i].isPseudoFunc;

×

2258

    pDummy[i].fpSet.init = pCtx[i].fpSet.init;

×

2260

×

2262

int32_t initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex,

×

2264

  SExecTaskInfo* pTaskInfo = downstream->pTaskInfo;

×

2265

  int32_t        code = TSDB_CODE_SUCCESS;

×

2266

  int32_t        lino = 0;

×

2267

  if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) {

×

2268

    SStreamPartitionOperatorInfo* pScanInfo = downstream->info;

×

2269

    pScanInfo->tsColIndex = tsColIndex;

×

2270

    pBasic->primaryPkIndex = pScanInfo->basic.primaryPkIndex;

×

2273

  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {

×

2274

    code = initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup, pBasic);

×

2275

    return code;

×

2277

  SStreamScanInfo* pScanInfo = downstream->info;

×

2278

  pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type};

×

2279

  pScanInfo->pState = pAggSup->pState;

×

2280

  if (!pScanInfo->pUpdateInfo) {

×

2281

    code = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark,

×

2282

                                              pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen,

×

2283

                                              &pScanInfo->pUpdateInfo);

×

2284

    QUERY_CHECK_CODE(code, lino, _end);

×

2286

  pScanInfo->twAggSup = *pTwSup;

×

2287

  pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo;

×

2288

  if (!hasSrcPrimaryKeyCol(pBasic)) {

×

2289

    pBasic->primaryPkIndex = pScanInfo->basic.primaryPkIndex;

×

2292

_end:

×

2293

  if (code != TSDB_CODE_SUCCESS) {

×

2294

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

2296

  return code;

×

2299

static TSKEY sesionTs(void* pKey) {

×

2300

  SSessionKey* pWinKey = (SSessionKey*)pKey;

×

2301

  return pWinKey->win.skey;

×

2304

int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, int32_t numOfOutput, int64_t gap,

×

2308

  pSup->resultRowSize = (keySize + getResultRowSize(pExpSup->pCtx, numOfOutput)) * ratio;

×

2309

  int32_t lino = 0;

×

2310

  int32_t code = createSpecialDataBlock(STREAM_CLEAR, &pSup->pScanBlock);

×

2311

  QUERY_CHECK_CODE(code, lino, _end);

×

2313

  pSup->gap = gap;

×

2314

  pSup->stateKeySize = keySize;

×

2315

  pSup->stateKeyType = keyType;

×

2316

  pSup->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfOutput, sizeof(SqlFunctionCtx));

×

2317

  QUERY_CHECK_NULL(pSup->pDummyCtx, code, lino, _end, terrno);

×

2319

  pSup->stateStore = *pStore;

×

2320

  pSup->pSessionAPI = pApi;

×

2322

  initDummyFunction(pSup->pDummyCtx, pExpSup->pCtx, numOfOutput);

×

2323

  pSup->pState = taosMemoryCalloc(1, sizeof(SStreamState));

×

2324

  QUERY_CHECK_NULL(pSup->pState, code, lino, _end, terrno);

×

2326

  *(pSup->pState) = *pState;

×

2327

  pSup->stateStore.streamStateSetNumber(pSup->pState, -1, tsIndex);

×

2328

  int32_t funResSize = getMaxFunResSize(pExpSup, numOfOutput);

×

2329

  if (stateType != STREAM_STATE_BUFF_HASH_SORT) {

×

2331

    pSup->pState->pResultRowStore.resultRowGet = getResultRowFromBuf;

×

2332

    pSup->pState->pResultRowStore.resultRowPut = putResultRowToBuf;

×

2333

    pSup->pState->pExprSupp = pExpSup;

×

2336

  if (stateType == STREAM_STATE_BUFF_SORT) {

×

2337

    pSup->pState->pFileState = NULL;

×

2338

    code = pSup->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SSessionKey), pSup->resultRowSize,

×

2339

                                                funResSize, sesionTs, pSup->pState, pTwAggSup->deleteMark, taskIdStr,

×

2340

                                                pHandle->checkpointId, stateType, &pSup->pState->pFileState);

×

2341

  } else if (stateType == STREAM_STATE_BUFF_HASH_SORT || stateType == STREAM_STATE_BUFF_HASH_SEARCH) {

×

2342

    pSup->pState->pFileState = NULL;

×

2343

    code = pSup->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pSup->resultRowSize, funResSize,

×

2344

                                                compareTs, pSup->pState, pTwAggSup->deleteMark, taskIdStr,

×

2345

                                                pHandle->checkpointId, stateType, &pSup->pState->pFileState);

×

2347

  QUERY_CHECK_CODE(code, lino, _end);

×

2349

  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

2350

  pSup->pResultRows = tSimpleHashInit(32, hashFn);

×

2351

  QUERY_CHECK_NULL(pSup->pResultRows, code, lino, _end, terrno);

×

2353

  for (int32_t i = 0; i < numOfOutput; ++i) {

×

2354

    pExpSup->pCtx[i].saveHandle.pState = pSup->pState;

×

2357

_end:

×

2358

  if (code != TSDB_CODE_SUCCESS) {

×

2359

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

2361

  return code;

×

2364

bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap) {

×

2365

  if (ts + gap >= pWin->skey && ts - gap <= pWin->ekey) {

×

2366

    return true;

×

2368

  return false;

×

2371

bool isInWindow(SResultWindowInfo* pWinInfo, TSKEY ts, int64_t gap) {

×

2372

  return isInTimeWindow(&pWinInfo->sessionWin.win, ts, gap);

×

2375

void getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,

×

2377

  pKey->win.skey = startTs;

×

2378

  pKey->win.ekey = endTs;

×

2379

  pKey->groupId = groupId;

×

2380

  int32_t code = pAggSup->stateStore.streamStateSessionGetKeyByRange(pAggSup->pState, pKey, pKey);

×

2381

  if (code != TSDB_CODE_SUCCESS) {

×

2382

    SET_SESSION_WIN_KEY_INVALID(pKey);

×

2384

×

2386

bool isInvalidSessionWin(SResultWindowInfo* pWinInfo) { return pWinInfo->sessionWin.win.skey == 0; }

×

2388

bool inWinRange(STimeWindow* range, STimeWindow* cur) {

×

2389

  if (cur->skey >= range->skey && cur->ekey <= range->ekey) {

×

2390

    return true;

×

2392

  return false;

×

2395

void clearOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) { pAPI->streamStateClearBuff(pState, pPos); }

×

2397

int32_t setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,

×

2399

  int32_t code = TSDB_CODE_SUCCESS;

×

2400

  int32_t lino = 0;

×

2401

  pCurWin->sessionWin.groupId = groupId;

×

2402

  pCurWin->sessionWin.win.skey = startTs;

×

2403

  pCurWin->sessionWin.win.ekey = endTs;

×

2404

  int32_t size = pAggSup->resultRowSize;

×

2405

  code = pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, pAggSup->gap,

×

2406

                                                             (void**)&pCurWin->pStatePos, &size, pWinCode);

×

2407

  QUERY_CHECK_CODE(code, lino, _end);

×

2409

  if (*pWinCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->sessionWin.win)) {

×

2410

    *pWinCode = TSDB_CODE_FAILED;

×

2411

    clearOutputBuf(pAggSup->pState, pCurWin->pStatePos, &pAggSup->pSessionAPI->stateStore);

×

2414

  if (*pWinCode == TSDB_CODE_SUCCESS) {

×

2415

    pCurWin->isOutput = true;

×

2416

    if (pCurWin->pStatePos->needFree) {

×

2417

      pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->sessionWin);

×

2420

    pCurWin->sessionWin.win.skey = startTs;

×

2421

    pCurWin->sessionWin.win.ekey = endTs;

×

2423

  qDebug("===stream===set session window buff .start:%" PRId64 ",end:%" PRId64 ",groupid:%" PRIu64,

×

2426

_end:

×

2427

  if (code != TSDB_CODE_SUCCESS) {

×

2428

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

2430

  return code;

×

2433

void getSessionWinBuf(SStreamAggSupporter* pAggSup, SStreamStateCur* pCur, SResultWindowInfo* pWinInfo,

×

2435

  int32_t size = 0;

×

2436

  (*pWinCode) = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin,

×

2437

                                                                 (void**)&pWinInfo->pStatePos, &size);

×

2438

  if ((*pWinCode) != TSDB_CODE_SUCCESS) {

×

2439

    return;

×

2442

  pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pCur);

×

2445

int32_t saveDeleteInfo(SArray* pWins, SSessionKey key) {

×

2446

  int32_t code = TSDB_CODE_SUCCESS;

×

2447

  int32_t lino = 0;

×

2448

  void*   res = taosArrayPush(pWins, &key);

×

2449

  if (!res) {

×

2450

    code = terrno;

×

2451

    QUERY_CHECK_CODE(code, lino, _end);

×

2454

_end:

×

2455

  if (code != TSDB_CODE_SUCCESS) {

×

2456

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

2458

  return code;

×

2461

int32_t saveDeleteRes(SSHashObj* pStDelete, SSessionKey key) {

×

2462

  key.win.ekey = key.win.skey;

×

2463

  return tSimpleHashPut(pStDelete, &key, sizeof(SSessionKey), NULL, 0);

×

2466

void releaseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) {

×

2467

  pAPI->streamStateReleaseBuf(pState, pPos, false);

×

2468

×

2470

void removeSessionResult(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey* pKey) {

×

2471

  int32_t     code = TSDB_CODE_SUCCESS;

×

2472

  int32_t     lino = 0;

×

2473

  SSessionKey key = {0};

×

2474

  getSessionHashKey(pKey, &key);

×

2475

  void* pVal = tSimpleHashGet(pHashMap, &key, sizeof(SSessionKey));

×

2476

  if (pVal) {

×

2477

    releaseOutputBuf(pAggSup->pState, *(void**)pVal, &pAggSup->pSessionAPI->stateStore);

×

2478

    int32_t tmpRes = tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey));

×

2479

    qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

2481

  int32_t tmpRes = tSimpleHashRemove(pResMap, &key, sizeof(SSessionKey));

×

2482

  qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

2483

×

2485

void getSessionHashKey(const SSessionKey* pKey, SSessionKey* pHashKey) {

×

2486

  *pHashKey = *pKey;

×

2487

  pHashKey->win.ekey = pKey->win.skey;

×

2488

×

2490

void removeSessionDeleteResults(SSHashObj* pHashMap, SArray* pWins) {

×

2491

  if (tSimpleHashGetSize(pHashMap) == 0) {

×

2492

    return;

×

2494

  int32_t size = taosArrayGetSize(pWins);

×

2495

  for (int32_t i = 0; i < size; i++) {

×

2496

    SResultWindowInfo* pWin = taosArrayGet(pWins, i);

×

2497

    if (!pWin) continue;

×

2498

    SSessionKey key = {0};

×

2499

    getSessionHashKey(&pWin->sessionWin, &key);

×

2500

    int32_t tmpRes = tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey));

×

2501

    qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

2505

void removeSessionResults(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SArray* pWins) {

×

2506

  if (tSimpleHashGetSize(pHashMap) == 0) {

×

2507

    return;

×

2509

  int32_t size = taosArrayGetSize(pWins);

×

2510

  for (int32_t i = 0; i < size; i++) {

×

2511

    SSessionKey* pWin = taosArrayGet(pWins, i);

×

2512

    if (!pWin) continue;

×

2513

    SSessionKey key = {0};

×

2514

    getSessionHashKey(pWin, &key);

×

2515

    void* pVal = tSimpleHashGet(pHashMap, &key, sizeof(SSessionKey));

×

2516

    if (pVal) {

×

2517

      releaseOutputBuf(pAggSup->pState, *(void**)pVal, &pAggSup->pSessionAPI->stateStore);

×

2518

      int32_t tmpRes = tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey));

×

2519

      qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

2524

int32_t updateSessionWindowInfo(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo, TSKEY* pStartTs,

×

2528

  int32_t code = TSDB_CODE_SUCCESS;

×

2529

  int32_t lino = 0;

×

2530

  for (int32_t i = start; i < rows; ++i) {

×

2531

    if (!isInWindow(pWinInfo, pStartTs[i], gap) && (!pEndTs || !isInWindow(pWinInfo, pEndTs[i], gap))) {

×

2532

      (*pWinRos) = i - start;

×

2533

      goto _end;

×

2535

    if (pWinInfo->sessionWin.win.skey > pStartTs[i]) {

×

2536

      if (pStDeleted && pWinInfo->isOutput) {

×

2537

        code = saveDeleteRes(pStDeleted, pWinInfo->sessionWin);

×

2538

        QUERY_CHECK_CODE(code, lino, _end);

×

2540

      removeSessionResult(pAggSup, pStUpdated, pResultRows, &pWinInfo->sessionWin);

×

2541

      pWinInfo->sessionWin.win.skey = pStartTs[i];

×

2543

    pWinInfo->sessionWin.win.ekey = TMAX(pWinInfo->sessionWin.win.ekey, pStartTs[i]);

×

2544

    if (pEndTs) {

×

2545

      pWinInfo->sessionWin.win.ekey = TMAX(pWinInfo->sessionWin.win.ekey, pEndTs[i]);

×

2547

    memcpy(pWinInfo->pStatePos->pKey, &pWinInfo->sessionWin, sizeof(SSessionKey));

×

2549

  (*pWinRos) = rows - start;

×

2551

_end:

×

2552

  if (code != TSDB_CODE_SUCCESS) {

×

2553

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

2555

  return code;

×

2558

static int32_t initSessionOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, SqlFunctionCtx* pCtx,

×

2560

  *pResult = (SResultRow*)pWinInfo->pStatePos->pRowBuff;

×

2562

  (*pResult)->win = pWinInfo->sessionWin.win;

×

2563

  return setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowEntryInfoOffset);

×

2566

int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindowInfo* pCurWin, SResultRow** pResult,

×

2569

  int32_t        code = TSDB_CODE_SUCCESS;

×

2570

  int32_t        lino = 0;

×

2571

  SExprSupp*     pSup = &pOperator->exprSupp;

×

2572

  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

×

2573

  code = initSessionOutputBuf(pCurWin, pResult, pSup->pCtx, numOutput, pSup->rowEntryInfoOffset);

×

2574

  QUERY_CHECK_CODE(code, lino, _end);

×

2576

  updateTimeWindowInfo(pTimeWindowData, &pCurWin->sessionWin.win, winDelta);

×

2577

  code = applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, pTimeWindowData, startIndex, winRows, rows, numOutput);

×

2579

_end:

×

2580

  if (code != TSDB_CODE_SUCCESS) {

×

2581

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

2583

  return code;

×

2586

void doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey) {

×

2587

  pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, pKey);

×

2588

  SSessionKey hashKey = {0};

×

2589

  getSessionHashKey(pKey, &hashKey);

×

2590

  int32_t tmpRes = tSimpleHashRemove(pAggSup->pResultRows, &hashKey, sizeof(SSessionKey));

×

2591

  qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

2592

×

2594

void setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo* pWinInfo) {

×

2595

  void* pVal = tSimpleHashGet(pStUpdated, &pWinInfo->sessionWin, sizeof(SSessionKey));

×

2596

  if (pVal) {

×

2597

    SResultWindowInfo* pWin = pVal;

×

2598

    pWinInfo->isOutput = pWin->isOutput;

×

2600

×

2602

void getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin,

×

2604

  SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->sessionWin);

×

2605

  pNextWin->isOutput = true;

×

2606

  setSessionWinOutputInfo(pStUpdated, pNextWin);

×

2607

  int32_t size = 0;

×

2608

  pNextWin->sessionWin = pCurWin->sessionWin;

×

2609

  int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->sessionWin,

×

2610

                                                                  (void**)&pNextWin->pStatePos, &size);

×

2611

  if (code != TSDB_CODE_SUCCESS) {

×

2612

    SET_SESSION_WIN_INVALID(*pNextWin);

×

2614

  pAggSup->stateStore.streamStateFreeCur(pCur);

×

2615

×

2617

int32_t compactTimeWindow(SExprSupp* pSup, SStreamAggSupporter* pAggSup, STimeWindowAggSupp* pTwAggSup,

×

2620

  int32_t     code = TSDB_CODE_SUCCESS;

×

2621

  int32_t     lino = 0;

×

2622

  SResultRow* pCurResult = NULL;

×

2623

  int32_t     numOfOutput = pSup->numOfExprs;

×

2624

  code = initSessionOutputBuf(pCurWin, &pCurResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset);

×

2625

  QUERY_CHECK_CODE(code, lino, _end);

×

2627

  SResultRow* pWinResult = NULL;

×

2628

  code = initSessionOutputBuf(pNextWin, &pWinResult, pAggSup->pDummyCtx, numOfOutput, pSup->rowEntryInfoOffset);

×

2629

  QUERY_CHECK_CODE(code, lino, _end);

×

2631

  pCurWin->sessionWin.win.ekey = TMAX(pCurWin->sessionWin.win.ekey, pNextWin->sessionWin.win.ekey);

×

2632

  memcpy(pCurWin->pStatePos->pKey, &pCurWin->sessionWin, sizeof(SSessionKey));

×

2634

  int64_t winDelta = 0;

×

2635

  if (addGap) {

×

2636

    winDelta = pAggSup->gap;

×

2638

  updateTimeWindowInfo(&pTwAggSup->timeWindowData, &pCurWin->sessionWin.win, winDelta);

×

2639

  code = compactFunctions(pSup->pCtx, pAggSup->pDummyCtx, numOfOutput, pTaskInfo, &pTwAggSup->timeWindowData);

×

2640

  QUERY_CHECK_CODE(code, lino, _end);

×

2642

  int32_t tmpRes = tSimpleHashRemove(pStUpdated, &pNextWin->sessionWin, sizeof(SSessionKey));

×

2643

  qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

2645

  if (pNextWin->isOutput && pStDeleted) {

×

2646

    qDebug("===stream=== save delete window info %" PRId64 ", %" PRIu64, pNextWin->sessionWin.win.skey,

×

2648

    code = saveDeleteRes(pStDeleted, pNextWin->sessionWin);

×

2649

    QUERY_CHECK_CODE(code, lino, _end);

×

2651

  removeSessionResult(pAggSup, pStUpdated, pAggSup->pResultRows, &pNextWin->sessionWin);

×

2652

  doDeleteSessionWindow(pAggSup, &pNextWin->sessionWin);

×

2653

  releaseOutputBuf(pAggSup->pState, pNextWin->pStatePos, &pAggSup->pSessionAPI->stateStore);

×

2655

_end:

×

2656

  if (code != TSDB_CODE_SUCCESS) {

×

2657

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

2659

  return code;

×

2662

static int32_t compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SSHashObj* pStUpdated,

×

2664

  int32_t                        code = TSDB_CODE_SUCCESS;

×

2665

  int32_t                        lino = 0;

×

2666

  SExprSupp*                     pSup = &pOperator->exprSupp;

×

2667

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

2668

  SStorageAPI*                   pAPI = &pOperator->pTaskInfo->storageAPI;

×

2669

  int32_t                        winNum = 0;

×

2670

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

2671

  SResultRow*                    pCurResult = NULL;

×

2672

  int32_t                        numOfOutput = pOperator->exprSupp.numOfExprs;

×

2673

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

2676

  while (1) {

×

2677

    SResultWindowInfo winInfo = {0};

×

2678

    getNextSessionWinInfo(pAggSup, pStUpdated, pCurWin, &winInfo);

×

2679

    if (!IS_VALID_SESSION_WIN(winInfo) || !isInWindow(pCurWin, winInfo.sessionWin.win.skey, pAggSup->gap) ||

×

2680

        !inWinRange(&pAggSup->winRange, &winInfo.sessionWin.win)) {

×

2681

      releaseOutputBuf(pAggSup->pState, winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);

×

2682

      break;

×

2685

        compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, pCurWin, &winInfo, pStUpdated, pStDeleted, true);

×

2686

    QUERY_CHECK_CODE(code, lino, _end);

×

2687

    winNum++;

×

2689

  if (pWinNum) {

×

2690

    (*pWinNum) = winNum;

×

2693

_end:

×

2694

  if (code != TSDB_CODE_SUCCESS) {

×

2695

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

2697

  return code;

×

2700

static void compactSessionSemiWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin) {

×

2701

  SExprSupp*                     pSup = &pOperator->exprSupp;

×

2702

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

2703

  SStorageAPI*                   pAPI = &pOperator->pTaskInfo->storageAPI;

×

2704

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

2705

  SResultRow*                    pCurResult = NULL;

×

2706

  int32_t                        numOfOutput = pOperator->exprSupp.numOfExprs;

×

2707

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

2709

  while (1) {

×

2710

    SResultWindowInfo winInfo = {0};

×

2711

    getNextSessionWinInfo(pAggSup, NULL, pCurWin, &winInfo);

×

2712

    if (!IS_VALID_SESSION_WIN(winInfo) || !isInWindow(pCurWin, winInfo.sessionWin.win.skey, pAggSup->gap) ||

×

2713

        !inWinRange(&pAggSup->winRange, &winInfo.sessionWin.win)) {

×

2714

      releaseOutputBuf(pAggSup->pState, winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);

×

2715

      break;

×

2717

    pCurWin->sessionWin.win.ekey = TMAX(pCurWin->sessionWin.win.ekey, winInfo.sessionWin.win.ekey);

×

2718

    memcpy(pCurWin->pStatePos->pKey, &pCurWin->sessionWin, sizeof(SSessionKey));

×

2719

    doDeleteSessionWindow(pAggSup, &winInfo.sessionWin);

×

2720

    releaseOutputBuf(pAggSup->pState, winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);

×

2722

×

2724

int32_t saveSessionOutputBuf(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo) {

×

2725

  qDebug("===stream===try save session result skey:%" PRId64 ", ekey:%" PRId64 ".pos%d", pWinInfo->sessionWin.win.skey,

×

2727

  return pAggSup->stateStore.streamStateSessionPut(pAggSup->pState, &pWinInfo->sessionWin, pWinInfo->pStatePos,

×

2731

static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pStUpdated,

×

2733

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

2734

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

2735

  int32_t                        numOfOutput = pOperator->exprSupp.numOfExprs;

×

2736

  uint64_t                       groupId = pSDataBlock->info.id.groupId;

×

2737

  int32_t                        code = TSDB_CODE_SUCCESS;

×

2738

  int32_t                        lino = 0;

×

2739

  SResultRow*                    pResult = NULL;

×

2740

  int32_t                        rows = pSDataBlock->info.rows;

×

2741

  int32_t                        winRows = 0;

×

2742

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

2744

  pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);

×

2745

  pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;

×

2746

  if (pAggSup->winRange.ekey <= 0) {

×

2747

    pAggSup->winRange.ekey = INT64_MAX;

×

2750

  SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);

×

2751

  TSKEY*           startTsCols = (int64_t*)pStartTsCol->pData;

×

2752

  SColumnInfoData* pEndTsCol = NULL;

×

2753

  if (hasEndTs) {

×

2754

    pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->endTsIndex);

×

2756

    pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);

×

2759

  TSKEY* endTsCols = (int64_t*)pEndTsCol->pData;

×

2761

  void*            pPkVal = NULL;

×

2762

  int32_t          pkLen = 0;

×

2763

  SColumnInfoData* pPkColDataInfo = NULL;

×

2764

  if (hasSrcPrimaryKeyCol(&pInfo->basic)) {

×

2765

    pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex);

×

2768

  for (int32_t i = 0; i < rows;) {

×

2769

    if (hasSrcPrimaryKeyCol(&pInfo->basic) && !IS_FINAL_SESSION_OP(pOperator) && pInfo->ignoreExpiredData) {

×

2770

      pPkVal = colDataGetData(pPkColDataInfo, i);

×

2771

      pkLen = colDataGetRowLength(pPkColDataInfo, i);

×

2773

    if (!IS_FINAL_SESSION_OP(pOperator) && pInfo->ignoreExpiredData &&

×

2774

        checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo, &pInfo->twAggSup,

×

2775

                         pSDataBlock->info.id.uid, endTsCols[i], pPkVal, pkLen)) {

×

2776

      i++;

×

2777

      continue;

×

2779

    SResultWindowInfo winInfo = {0};

×

2780

    int32_t           winCode = TSDB_CODE_SUCCESS;

×

2781

    code = setSessionOutputBuf(pAggSup, startTsCols[i], endTsCols[i], groupId, &winInfo, &winCode);

×

2782

    QUERY_CHECK_CODE(code, lino, _end);

×

2784

    if (winCode != TSDB_CODE_SUCCESS && IS_NORMAL_SESSION_OP(pOperator) &&

×

2785

        BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {

×

2786

      code = addSessionAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &winInfo.sessionWin, &pInfo->basic.notifyEventSup,

×

2788

      QUERY_CHECK_CODE(code, lino, _end);

×

2792

    if (!winInfo.pStatePos) {

×

2793

      continue;

×

2795

    setSessionWinOutputInfo(pStUpdated, &winInfo);

×

2796

    code = updateSessionWindowInfo(pAggSup, &winInfo, startTsCols, endTsCols, groupId, rows, i, pAggSup->gap,

×

2798

    QUERY_CHECK_CODE(code, lino, _end);

×

2800

    int64_t winDelta = 0;

×

2801

    if (addGap) {

×

2802

      winDelta = pAggSup->gap;

×

2804

    code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &winInfo, &pResult, i, winRows, rows, numOfOutput,

×

2806

    QUERY_CHECK_CODE(code, lino, _end);

×

2808

    code = compactSessionWindow(pOperator, &winInfo, pStUpdated, pStDeleted, addGap, NULL);

×

2809

    QUERY_CHECK_CODE(code, lino, _end);

×

2811

    code = saveSessionOutputBuf(pAggSup, &winInfo);

×

2812

    QUERY_CHECK_CODE(code, lino, _end);

×

2814

    if (pInfo->destHasPrimaryKey && winInfo.isOutput && IS_NORMAL_SESSION_OP(pOperator)) {

×

2815

      code = saveDeleteRes(pInfo->pPkDeleted, winInfo.sessionWin);

×

2816

      QUERY_CHECK_CODE(code, lino, _end);

×

2818

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) {

×

2819

      code = saveResult(winInfo, pStUpdated);

×

2820

      QUERY_CHECK_CODE(code, lino, _end);

×

2822

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

2823

      winInfo.pStatePos->beUpdated = true;

×

2824

      SSessionKey key = {0};

×

2825

      getSessionHashKey(&winInfo.sessionWin, &key);

×

2826

      code = tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo));

×

2827

      QUERY_CHECK_CODE(code, lino, _end);

×

2830

    i += winRows;

×

2833

_end:

×

2834

  if (code != TSDB_CODE_SUCCESS) {

×

2835

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

2837

×

2839

int32_t doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result) {

×

2840

  int32_t          code = TSDB_CODE_SUCCESS;

×

2841

  int32_t          lino = 0;

×

2842

  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);

×

2843

  TSKEY*           startDatas = (TSKEY*)pStartTsCol->pData;

×

2844

  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);

×

2845

  TSKEY*           endDatas = (TSKEY*)pEndTsCol->pData;

×

2846

  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);

×

2847

  uint64_t*        gpDatas = (uint64_t*)pGroupCol->pData;

×

2848

  for (int32_t i = 0; i < pBlock->info.rows; i++) {

×

2849

    while (1) {

×

2850

      SSessionKey curWin = {0};

×

2851

      getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], &curWin);

×

2852

      if (IS_INVALID_SESSION_WIN_KEY(curWin)) {

×

2853

        break;

×

2855

      doDeleteSessionWindow(pAggSup, &curWin);

×

2856

      if (result) {

×

2857

        code = saveDeleteInfo(result, curWin);

×

2858

        QUERY_CHECK_CODE(code, lino, _end);

×

2863

_end:

×

2864

  if (code != TSDB_CODE_SUCCESS) {

×

2865

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

2867

  return code;

×

2870

inline int32_t sessionKeyCompareAsc(const void* pKey1, const void* pKey2) {

×

2871

  SResultWindowInfo* pWinInfo1 = (SResultWindowInfo*)pKey1;

×

2872

  SResultWindowInfo* pWinInfo2 = (SResultWindowInfo*)pKey2;

×

2873

  SSessionKey*       pWin1 = &pWinInfo1->sessionWin;

×

2874

  SSessionKey*       pWin2 = &pWinInfo2->sessionWin;

×

2876

  if (pWin1->groupId > pWin2->groupId) {

×

2877

    return 1;

×

2878

  } else if (pWin1->groupId < pWin2->groupId) {

×

2879

    return -1;

×

2882

  if (pWin1->win.skey > pWin2->win.skey) {

×

2883

    return 1;

×

2884

  } else if (pWin1->win.skey < pWin2->win.skey) {

×

2885

    return -1;

×

2888

  return 0;

×

2891

static int32_t appendToDeleteDataBlock(SOperatorInfo* pOp, SSDataBlock *pBlock, SSessionKey *pKey) {

×

2892

  int32_t        code = TSDB_CODE_SUCCESS;

×

2893

  int32_t        lino = 0;

×

2894

  SExecTaskInfo* pTaskInfo = pOp->pTaskInfo;

×

2896

  QUERY_CHECK_NULL(pBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);

×

2897

  QUERY_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA);

×

2899

  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);

×

2900

  code = colDataSetVal(pStartTsCol, pBlock->info.rows, (const char*)&pKey->win.skey, false);

×

2901

  QUERY_CHECK_CODE(code, lino, _end);

×

2903

  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);

×

2904

  code = colDataSetVal(pEndTsCol, pBlock->info.rows, (const char*)&pKey->win.skey, false);

×

2905

  QUERY_CHECK_CODE(code, lino, _end);

×

2907

  SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);

×

2908

  colDataSetNULL(pUidCol, pBlock->info.rows);

×

2910

  SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);

×

2911

  code = colDataSetVal(pGpCol, pBlock->info.rows, (const char*)&pKey->groupId, false);

×

2912

  QUERY_CHECK_CODE(code, lino, _end);

×

2914

  SColumnInfoData* pCalStCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);

×

2915

  colDataSetNULL(pCalStCol, pBlock->info.rows);

×

2917

  SColumnInfoData* pCalEdCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);

×

2918

  colDataSetNULL(pCalEdCol, pBlock->info.rows);

×

2920

  SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);

×

2921

  if (!pTableCol) {

×

2922

    QUERY_CHECK_CODE(code, lino, _end);

×

2925

  void*        tbname = NULL;

×

2926

  int32_t      winCode = TSDB_CODE_SUCCESS;

×

2927

  SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI;

×

2929

      pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, pKey->groupId, &tbname, false, &winCode);

×

2930

  QUERY_CHECK_CODE(code, lino, _end);

×

2932

  if (winCode != TSDB_CODE_SUCCESS) {

×

2933

    colDataSetNULL(pTableCol, pBlock->info.rows);

×

2936

    STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));

×

2937

    code = colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false);

×

2938

    QUERY_CHECK_CODE(code, lino, _end);

×

2939

    pAPI->stateStore.streamStateFreeVal(tbname);

×

2941

  pBlock->info.rows += 1;

×

2943

_end:

×

2944

  if (code != TSDB_CODE_SUCCESS) {

×

2945

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

2947

  return code;

×

2950

void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite,

×

2952

  int32_t        code = TSDB_CODE_SUCCESS;

×

2953

  int32_t        lino = 0;

×

2954

  SExecTaskInfo* pTaskInfo = pOp->pTaskInfo;

×

2955

  int64_t        minWindowSize = getMinWindowSize(pOp);

×

2956

  int32_t        numOfRows = getNumOfTotalRes(pGroupResInfo);

×

2958

  blockDataCleanup(pBlock);

×

2959

  int32_t size = tSimpleHashGetSize(pStDeleted);

×

2960

  if (minWindowSize > 0) {

×

2962

    for (int32_t i = pGroupResInfo->delIndex; i < numOfRows; ++i) {

×

2963

      SResultWindowInfo* pWinInfo = taosArrayGet(pGroupResInfo->pRows, i);

×

2964

      SRowBuffPos*       pPos = pWinInfo->pStatePos;

×

2965

      SSessionKey*       pKey = (SSessionKey*)pPos->pKey;

×

2966

      if (pKey->win.ekey - pKey->win.skey < minWindowSize) {

×

2967

        size++;

×

2971

  if (size == 0) {

×

2972

    return;

×

2974

  code = blockDataEnsureCapacity(pBlock, size);

×

2975

  QUERY_CHECK_CODE(code, lino, _end);

×

2977

  int32_t iter = 0;

×

2978

  while (((*Ite) = tSimpleHashIterate(pStDeleted, *Ite, &iter)) != NULL) {

×

2979

    if (pBlock->info.rows + 1 > pBlock->info.capacity) {

×

2980

      break;

×

2982

    SSessionKey*     res = tSimpleHashGetKey(*Ite, NULL);

×

2983

    code = appendToDeleteDataBlock(pOp, pBlock, res);

×

2984

    QUERY_CHECK_CODE(code, lino, _end);

×

2987

  if (minWindowSize > 0) {

×

2988

    for (int32_t i = pGroupResInfo->delIndex; i < numOfRows; ++i) {

×

2989

      SResultWindowInfo* pWinInfo = taosArrayGet(pGroupResInfo->pRows, i);

×

2990

      SRowBuffPos*       pPos = pWinInfo->pStatePos;

×

2991

      SSessionKey*       pKey = (SSessionKey*)pPos->pKey;

×

2992

      if (pKey->win.ekey - pKey->win.skey < minWindowSize) {

×

2993

        code = appendToDeleteDataBlock(pOp, pBlock, pKey);

×

2994

        QUERY_CHECK_CODE(code, lino, _end);

×

2997

    pGroupResInfo->delIndex = numOfRows;

×

3000

_end:

×

3001

  if ((*Ite) == NULL) {

×

3002

    tSimpleHashClear(pStDeleted);

×

3005

  if (code != TSDB_CODE_SUCCESS) {

×

3006

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3010

static int32_t rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SSHashObj* pStUpdated) {

×

3011

  int32_t        winCode = TSDB_CODE_SUCCESS;

×

3012

  int32_t        code = TSDB_CODE_SUCCESS;

×

3013

  int32_t        lino = 0;

×

3014

  SExprSupp*     pSup = &pOperator->exprSupp;

×

3015

  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

×

3016

  SStorageAPI*   pAPI = &pOperator->pTaskInfo->storageAPI;

×

3018

  int32_t                        size = taosArrayGetSize(pWinArray);

×

3019

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3020

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

3021

  int32_t                        numOfOutput = pSup->numOfExprs;

×

3022

  int32_t                        numOfChild = taosArrayGetSize(pInfo->pChildren);

×

3024

  for (int32_t i = 0; i < size; i++) {

×

3025

    SSessionKey*      pWinKey = taosArrayGet(pWinArray, i);

×

3026

    int32_t           num = 0;

×

3027

    SResultWindowInfo parentWin = {0};

×

3028

    for (int32_t j = 0; j < numOfChild; j++) {

×

3029

      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, j);

×

3030

      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;

×

3031

      SStreamAggSupporter*           pChAggSup = &pChInfo->streamAggSup;

×

3032

      SSessionKey                    chWinKey = {0};

×

3033

      getSessionHashKey(pWinKey, &chWinKey);

×

3034

      SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pChAggSup->pState, &chWinKey);

×

3035

      SResultRow*      pResult = NULL;

×

3036

      SResultRow*      pChResult = NULL;

×

3037

      while (1) {

×

3038

        SResultWindowInfo childWin = {0};

×

3039

        childWin.sessionWin = *pWinKey;

×

3040

        getSessionWinBuf(pChAggSup, pCur, &childWin, &winCode);

×

3042

        if (winCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &childWin.sessionWin.win)) {

×

3043

          releaseOutputBuf(pAggSup->pState, childWin.pStatePos, &pAggSup->stateStore);

×

3044

          continue;

×

3047

        if (winCode == TSDB_CODE_SUCCESS && inWinRange(&pWinKey->win, &childWin.sessionWin.win)) {

×

3048

          if (num == 0) {

×

3049

            code = setSessionOutputBuf(pAggSup, pWinKey->win.skey, pWinKey->win.ekey, pWinKey->groupId, &parentWin,

×

3051

            QUERY_CHECK_CODE(code, lino, _end);

×

3053

            parentWin.sessionWin = childWin.sessionWin;

×

3054

            memcpy(parentWin.pStatePos->pKey, &parentWin.sessionWin, sizeof(SSessionKey));

×

3055

            code = initSessionOutputBuf(&parentWin, &pResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset);

×

3056

            QUERY_CHECK_CODE(code, lino, _end);

×

3058

          num++;

×

3059

          parentWin.sessionWin.win.skey = TMIN(parentWin.sessionWin.win.skey, childWin.sessionWin.win.skey);

×

3060

          parentWin.sessionWin.win.ekey = TMAX(parentWin.sessionWin.win.ekey, childWin.sessionWin.win.ekey);

×

3061

          memcpy(parentWin.pStatePos->pKey, &parentWin.sessionWin, sizeof(SSessionKey));

×

3063

          updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin.sessionWin.win, pAggSup->gap);

×

3064

          code = initSessionOutputBuf(&childWin, &pChResult, pChild->exprSupp.pCtx, numOfOutput,

×

3066

          QUERY_CHECK_CODE(code, lino, _end);

×

3068

          code = compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo,

×

3070

          QUERY_CHECK_CODE(code, lino, _end);

×

3072

          code = compactSessionWindow(pOperator, &parentWin, pStUpdated, NULL, true, NULL);

×

3073

          QUERY_CHECK_CODE(code, lino, _end);

×

3075

          releaseOutputBuf(pAggSup->pState, childWin.pStatePos, &pAggSup->stateStore);

×

3077

          releaseOutputBuf(pAggSup->pState, childWin.pStatePos, &pAggSup->stateStore);

×

3078

          break;

×

3081

      pAPI->stateStore.streamStateFreeCur(pCur);

×

3083

    if (num > 0) {

×

3084

      code = saveResult(parentWin, pStUpdated);

×

3085

      QUERY_CHECK_CODE(code, lino, _end);

×

3087

      code = saveSessionOutputBuf(pAggSup, &parentWin);

×

3088

      QUERY_CHECK_CODE(code, lino, _end);

×

3092

_end:

×

3093

  if (code != TSDB_CODE_SUCCESS) {

×

3094

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3096

  return code;

×

3099

int32_t closeSessionWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SSHashObj* pClosed) {

×

3100

  int32_t code = TSDB_CODE_SUCCESS;

×

3101

  int32_t lino = 0;

×

3102

  void*   pIte = NULL;

×

3103

  int32_t iter = 0;

×

3104

  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {

×

3105

    SResultWindowInfo* pWinInfo = pIte;

×

3106

    if (isCloseWindow(&pWinInfo->sessionWin.win, pTwSup)) {

×

3107

      if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE && pClosed) {

×

3108

        code = saveResult(*pWinInfo, pClosed);

×

3109

        QUERY_CHECK_CODE(code, lino, _end);

×

3111

      SSessionKey* pKey = tSimpleHashGetKey(pIte, NULL);

×

3112

      code = tSimpleHashIterateRemove(pHashMap, pKey, sizeof(SSessionKey), &pIte, &iter);

×

3113

      QUERY_CHECK_CODE(code, lino, _end);

×

3116

_end:

×

3117

  if (code != TSDB_CODE_SUCCESS) {

×

3118

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3120

  return code;

×

3123

static int32_t closeChildSessionWindow(SArray* pChildren, TSKEY maxTs) {

×

3124

  int32_t code = TSDB_CODE_SUCCESS;

×

3125

  int32_t lino = 0;

×

3127

  int32_t size = taosArrayGetSize(pChildren);

×

3128

  for (int32_t i = 0; i < size; i++) {

×

3129

    SOperatorInfo*                 pChildOp = taosArrayGetP(pChildren, i);

×

3130

    SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info;

×

3131

    pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);

×

3132

    code = closeSessionWindow(pChInfo->streamAggSup.pResultRows, &pChInfo->twAggSup, NULL);

×

3133

    QUERY_CHECK_CODE(code, lino, _end);

×

3135

_end:

×

3136

  if (code != TSDB_CODE_SUCCESS) {

×

3137

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3139

  return code;

×

3142

int32_t getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated) {

×

3143

  int32_t code = TSDB_CODE_SUCCESS;

×

3144

  int32_t lino = 0;

×

3145

  void*   pIte = NULL;

×

3146

  int32_t iter = 0;

×

3147

  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {

×

3148

    SResultWindowInfo* pWinInfo = pIte;

×

3149

    if (!pWinInfo->pStatePos->beUpdated) {

×

3150

      continue;

×

3152

    pWinInfo->pStatePos->beUpdated = false;

×

3153

    code = saveResult(*pWinInfo, pStUpdated);

×

3154

    QUERY_CHECK_CODE(code, lino, _end);

×

3157

_end:

×

3158

  if (code != TSDB_CODE_SUCCESS) {

×

3159

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3161

  return code;

×

3164

int32_t copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted) {

×

3165

  int32_t code = TSDB_CODE_SUCCESS;

×

3166

  int32_t lino = 0;

×

3167

  int32_t size = taosArrayGetSize(pResWins);

×

3168

  for (int32_t i = 0; i < size; i++) {

×

3169

    SSessionKey* pWinKey = taosArrayGet(pResWins, i);

×

3170

    if (!pWinKey) continue;

×

3171

    SSessionKey winInfo = {0};

×

3172

    getSessionHashKey(pWinKey, &winInfo);

×

3173

    code = tSimpleHashPut(pStDeleted, &winInfo, sizeof(SSessionKey), NULL, 0);

×

3174

    QUERY_CHECK_CODE(code, lino, _end);

×

3177

_end:

×

3178

  if (code != TSDB_CODE_SUCCESS) {

×

3179

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3181

  return code;

×

3185

void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList) {

×

3186

  pGroupResInfo->pRows = pArrayList;

×

3187

  pGroupResInfo->index = 0;

×

3188

  pGroupResInfo->pBuf = NULL;

×

3189

  pGroupResInfo->freeItem = false;

×

3190

  pGroupResInfo->delIndex = 0;

×

3191

×

3193

int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup,

×

3195

  int32_t         code = TSDB_CODE_SUCCESS;

×

3196

  int32_t         lino = 0;

×

3197

  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;

×

3198

  SStorageAPI*    pAPI = &pTaskInfo->storageAPI;

×

3199

  SExprInfo*      pExprInfo = pSup->pExprInfo;

×

3200

  int32_t         numOfExprs = pSup->numOfExprs;

×

3201

  int32_t*        rowEntryOffset = pSup->rowEntryInfoOffset;

×

3202

  SqlFunctionCtx* pCtx = pSup->pCtx;

×

3203

  int64_t         minWindowSize = getMinWindowSize(pOperator);

×

3205

  int32_t numOfRows = getNumOfTotalRes(pGroupResInfo);

×

3207

  for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) {

×

3208

    SResultWindowInfo* pWinInfo = taosArrayGet(pGroupResInfo->pRows, i);

×

3209

    SRowBuffPos*       pPos = pWinInfo->pStatePos;

×

3210

    SResultRow*        pRow = NULL;

×

3211

    SSessionKey*       pKey = (SSessionKey*)pPos->pKey;

×

3213

    if (pBlock->info.id.groupId == 0) {

×

3214

      pBlock->info.id.groupId = pKey->groupId;

×

3216

      void*   tbname = NULL;

×

3217

      int32_t winCode = TSDB_CODE_SUCCESS;

×

3218

      code = pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId,

×

3220

      QUERY_CHECK_CODE(code, lino, _end);

×

3222

      if (winCode != TSDB_CODE_SUCCESS) {

×

3223

        pBlock->info.parTbName[0] = 0;

×

3225

        memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);

×

3227

      pAPI->stateStore.streamStateFreeVal(tbname);

×

3230

      if (pBlock->info.id.groupId != pKey->groupId) {

×

3231

        break;

×

3235

    code = pAPI->stateStore.streamStateGetByPos(pState, pPos, (void**)&pRow);

×

3236

    QUERY_CHECK_CODE(code, lino, _end);

×

3238

    doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset);

×

3240

    if (pRow->numOfRows == 0) {

×

3241

      pGroupResInfo->index += 1;

×

3242

      continue;

×

3245

    if (pKey->win.ekey - pKey->win.skey < minWindowSize) {

×

3246

      qDebug("skip small window, groupId: %" PRId64 ", windowSize: %" PRId64 ", minWindowSize: %" PRId64, pKey->groupId,

×

3248

      pGroupResInfo->index += 1;

×

3249

      continue;

×

3252

    if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) {

×

3253

      break;

×

3256

    pGroupResInfo->index += 1;

×

3258

    for (int32_t j = 0; j < numOfExprs; ++j) {

×

3259

      int32_t slotId = pExprInfo[j].base.resSchema.slotId;

×

3261

      pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset);

×

3262

      if (pCtx[j].fpSet.finalize) {

×

3263

        int32_t tmpRes = pCtx[j].fpSet.finalize(&pCtx[j], pBlock);

×

3264

        if (TAOS_FAILED(tmpRes)) {

×

3265

          qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(tmpRes));

×

3266

          QUERY_CHECK_CODE(code, lino, _end);

×

3268

      } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) {

×

3273

        SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId);

×

3274

        char*            in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo);

×

3275

        for (int32_t k = 0; k < pRow->numOfRows; ++k) {

×

3276

          code = colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes);

×

3277

          QUERY_CHECK_CODE(code, lino, _end);

×

3282

    if (pSessionKeys) {

×

3283

      for (int32_t j = 0; j < pRow->numOfRows; ++j) {

×

3284

        const void* px = taosArrayPush(pSessionKeys, pKey);

×

3285

        QUERY_CHECK_NULL(px, code, lino, _end, terrno);

×

3289

    pBlock->info.dataLoad = 1;

×

3290

    pBlock->info.rows += pRow->numOfRows;

×

3292

  code = blockDataUpdateTsWindow(pBlock, 0);

×

3293

  QUERY_CHECK_CODE(code, lino, _end);

×

3295

_end:

×

3296

  if (code != TSDB_CODE_SUCCESS) {

×

3297

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3299

  return code;

×

3302

void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo, SSDataBlock* pBlock,

×

3304

  int32_t        code = TSDB_CODE_SUCCESS;

×

3305

  int32_t        lino = 0;

×

3306

  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

×

3308

  pBlock->info.version = pTaskInfo->version;

×

3310

  blockDataCleanup(pBlock);

×

3311

  taosArrayClear(pSessionKeys);

×

3312

  if (!hasRemainResults(pGroupResInfo)) {

×

3313

    cleanupGroupResInfo(pGroupResInfo);

×

3314

    goto _end;

×

3318

  pBlock->info.id.groupId = 0;

×

3319

  code = buildSessionResultDataBlock(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo, pSessionKeys);

×

3320

  QUERY_CHECK_CODE(code, lino, _end);

×

3322

  if (pBlock->info.rows == 0) {

×

3323

    cleanupGroupResInfo(pGroupResInfo);

×

3326

_end:

×

3327

  if (code != TSDB_CODE_SUCCESS) {

×

3328

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3330

×

3332

static int32_t buildSessionResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

3333

  int32_t                        code = TSDB_CODE_SUCCESS;

×

3334

  int32_t                        lino = 0;

×

3335

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3336

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

3337

  SOptrBasicInfo*                pBInfo = &pInfo->binfo;

×

3338

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

3339

  SStreamNotifyEventSupp*        pNotifySup = &pInfo->basic.notifyEventSup;

×

3340

  STaskNotifyEventStat*          pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;

×

3341

  bool                           addNotifyEvent = false;

×

3342

  addNotifyEvent = IS_NORMAL_SESSION_OP(pOperator) &&

×

3343

                   BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);

×

3344

  doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator, &pInfo->groupResInfo);

×

3345

  if (pInfo->pDelRes->info.rows > 0) {

×

3346

    printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

3347

    if (addNotifyEvent) {

×

3348

      code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);

×

3349

      QUERY_CHECK_CODE(code, lino, _end);

×

3351

    (*ppRes) = pInfo->pDelRes;

×

3352

    return code;

×

3355

  doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes,

×

3357

  if (pBInfo->pRes->info.rows > 0) {

×

3358

    printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

3359

    if (addNotifyEvent) {

×

3361

      for (int32_t i = 0; i < taosArrayGetSize(pNotifySup->pSessionKeys); ++i) {

×

3362

        SSessionKey* pKey = taosArrayGet(pNotifySup->pSessionKeys, i);

×

3363

        pKey->win.ekey += pAggSup->gap;

×

3365

      code = addAggResultNotifyEvent(pBInfo->pRes, pNotifySup->pSessionKeys, pTaskInfo->streamInfo.notifyResultSchema,

×

3367

      QUERY_CHECK_CODE(code, lino, _end);

×

3369

    (*ppRes) = pBInfo->pRes;

×

3370

    return code;

×

3373

  code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);

×

3374

  QUERY_CHECK_CODE(code, lino, _end);

×

3375

  if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {

×

3376

    printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

3377

    (*ppRes) = pNotifySup->pEventBlock;

×

3378

    return code;

×

3381

  code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);

×

3382

  QUERY_CHECK_CODE(code, lino, _end);

×

3384

_end:

×

3385

  if (code != TSDB_CODE_SUCCESS) {

×

3386

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3388

  (*ppRes) = NULL;

×

3389

  return code;

×

3392

int32_t getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins) {

×

3393

  int32_t code = TSDB_CODE_SUCCESS;

×

3394

  int32_t lino = 0;

×

3395

  int32_t size = taosArrayGetSize(pAllWins);

×

3396

  if (size == 0) {

×

3397

    goto _end;

×

3399

  SResultWindowInfo* pWinInfo = taosArrayGet(pAllWins, size - 1);

×

3400

  SSessionKey*       pSeKey = &pWinInfo->sessionWin;

×

3401

  void*              tmp = taosArrayPush(pMaxWins, pSeKey);

×

3402

  if (!tmp) {

×

3403

    code = terrno;

×

3404

    QUERY_CHECK_CODE(code, lino, _end);

×

3407

  if (pSeKey->groupId == 0) {

×

3408

    goto _end;

×

3410

  uint64_t preGpId = pSeKey->groupId;

×

3411

  for (int32_t i = size - 2; i >= 0; i--) {

×

3412

    pWinInfo = taosArrayGet(pAllWins, i);

×

3413

    pSeKey = &pWinInfo->sessionWin;

×

3414

    if (preGpId != pSeKey->groupId) {

×

3415

      void* tmp = taosArrayPush(pMaxWins, pSeKey);

×

3416

      if (!tmp) {

×

3417

        code = terrno;

×

3418

        QUERY_CHECK_CODE(code, lino, _end);

×

3420

      preGpId = pSeKey->groupId;

×

3424

_end:

×

3425

  if (code != TSDB_CODE_SUCCESS) {

×

3426

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3428

  return code;

×

3431

int32_t encodeSSessionKey(void** buf, SSessionKey* key) {

×

3432

  int32_t tlen = 0;

×

3433

  tlen += encodeSTimeWindow(buf, &key->win);

×

3434

  tlen += taosEncodeFixedU64(buf, key->groupId);

×

3435

  return tlen;

×

3438

void* decodeSSessionKey(void* buf, SSessionKey* key) {

×

3439

  buf = decodeSTimeWindow(buf, &key->win);

×

3440

  buf = taosDecodeFixedU64(buf, &key->groupId);

×

3441

  return buf;

×

3444

int32_t encodeSResultWindowInfo(void** buf, SResultWindowInfo* key, int32_t outLen) {

×

3445

  int32_t tlen = 0;

×

3446

  tlen += taosEncodeFixedBool(buf, key->isOutput);

×

3447

  tlen += encodeSSessionKey(buf, &key->sessionWin);

×

3448

  return tlen;

×

3451

void* decodeSResultWindowInfo(void* buf, SResultWindowInfo* key, int32_t outLen) {

×

3452

  buf = taosDecodeFixedBool(buf, &key->isOutput);

×

3453

  buf = decodeSSessionKey(buf, &key->sessionWin);

×

3454

  return buf;

×

3457

int32_t doStreamSessionEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOperator, bool isParent) {

×

3458

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3459

  if (!pInfo) {

×

3460

    return 0;

×

3463

  void* pData = (buf == NULL) ? NULL : *buf;

×

3466

  int32_t tlen = 0;

×

3467

  int32_t mapSize = tSimpleHashGetSize(pInfo->streamAggSup.pResultRows);

×

3468

  tlen += taosEncodeFixedI32(buf, mapSize);

×

3469

  void*   pIte = NULL;

×

3470

  size_t  keyLen = 0;

×

3471

  int32_t iter = 0;

×

3472

  while ((pIte = tSimpleHashIterate(pInfo->streamAggSup.pResultRows, pIte, &iter)) != NULL) {

×

3473

    void* key = tSimpleHashGetKey(pIte, &keyLen);

×

3474

    tlen += encodeSSessionKey(buf, key);

×

3475

    tlen += encodeSResultWindowInfo(buf, pIte, pInfo->streamAggSup.resultRowSize);

×

3479

  tlen += encodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);

×

3482

  int32_t size = taosArrayGetSize(pInfo->pChildren);

×

3483

  tlen += taosEncodeFixedI32(buf, size);

×

3484

  for (int32_t i = 0; i < size; i++) {

×

3485

    SOperatorInfo* pChOp = taosArrayGetP(pInfo->pChildren, i);

×

3486

    tlen += doStreamSessionEncodeOpState(buf, 0, pChOp, false);

×

3490

  tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);

×

3493

  tlen += encodeStreamBasicInfo(buf, &pInfo->basic);

×

3496

  if (isParent) {

×

3497

    if (buf) {

×

3498

      uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));

×

3499

      tlen += taosEncodeFixedU32(buf, cksum);

×

3501

      tlen += sizeof(uint32_t);

×

3505

  return tlen;

×

3508

int32_t doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator, bool isParent, void** ppBuf) {

×

3509

  int32_t                        code = TSDB_CODE_SUCCESS;

×

3510

  int32_t                        lino = 0;

×

3511

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3512

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

3513

  void*                        pDataEnd = POINTER_SHIFT(buf, len);

×

3514

  if (!pInfo) {

×

3515

    code = TSDB_CODE_FAILED;

×

3516

    QUERY_CHECK_CODE(code, lino, _end);

×

3518

  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;

×

3521

  if (isParent) {

×

3522

    int32_t dataLen = len - sizeof(uint32_t);

×

3523

    void*   pCksum = POINTER_SHIFT(buf, dataLen);

×

3524

    if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) {

×

3525

      qError("stream session state is invalid");

×

3526

      code = TSDB_CODE_FAILED;

×

3527

      QUERY_CHECK_CODE(code, lino, _end);

×

3529

    pDataEnd = pCksum;

×

3533

  int32_t mapSize = 0;

×

3534

  buf = taosDecodeFixedI32(buf, &mapSize);

×

3535

  for (int32_t i = 0; i < mapSize; i++) {

×

3536

    SResultWindowInfo winfo = {0};

×

3537

    buf = decodeSSessionKey(buf, &winfo.sessionWin);

×

3538

    int32_t winCode = TSDB_CODE_SUCCESS;

×

3539

    code = pAggSup->stateStore.streamStateSessionAddIfNotExist(

×

3541

    QUERY_CHECK_CODE(code, lino, _end);

×

3542

    QUERY_CHECK_CONDITION((winCode == TSDB_CODE_SUCCESS), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);

×

3544

    buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize);

×

3545

    code = tSimpleHashPut(pInfo->streamAggSup.pResultRows, &winfo.sessionWin, sizeof(SSessionKey), &winfo,

×

3547

    QUERY_CHECK_CODE(code, lino, _end);

×

3551

  buf = decodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);

×

3554

  int32_t size = 0;

×

3555

  buf = taosDecodeFixedI32(buf, &size);

×

3556

  for (int32_t i = 0; i < size; i++) {

×

3557

    SOperatorInfo* pChOp = taosArrayGetP(pInfo->pChildren, i);

×

3558

    code = doStreamSessionDecodeOpState(buf, 0, pChOp, false, &buf);

×

3559

    QUERY_CHECK_CODE(code, lino, _end);

×

3563

  buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);

×

3564

  if (ppBuf) {

×

3565

    (*ppBuf) = buf;

×

3569

  if (buf < pDataEnd) {

×

3570

    code = decodeStreamBasicInfo(&buf, &pInfo->basic);

×

3571

    QUERY_CHECK_CODE(code, lino, _end);

×

3574

_end:

×

3575

  if (code != TSDB_CODE_SUCCESS) {

×

3576

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3578

  return code;

×

3581

void doStreamSessionSaveCheckpoint(SOperatorInfo* pOperator) {

×

3582

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3583

  if (needSaveStreamOperatorInfo(&pInfo->basic)) {

×

3584

    int32_t len = doStreamSessionEncodeOpState(NULL, 0, pOperator, true);

×

3585

    void*   buf = taosMemoryCalloc(1, len);

×

3586

    if (!buf) {

×

3587

      qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY));

×

3588

      return;

×

3590

    void* pBuf = buf;

×

3591

    len = doStreamSessionEncodeOpState(&pBuf, len, pOperator, true);

×

3592

    pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME,

×

3594

    taosMemoryFree(buf);

×

3595

    saveStreamOperatorStateComplete(&pInfo->basic);

×

3599

void resetUnCloseSessionWinInfo(SSHashObj* winMap) {

×

3600

  void*   pIte = NULL;

×

3601

  int32_t iter = 0;

×

3602

  while ((pIte = tSimpleHashIterate(winMap, pIte, &iter)) != NULL) {

×

3603

    SResultWindowInfo* pResInfo = pIte;

×

3604

    pResInfo->pStatePos->beUsed = true;

×

3606

×

3608

int32_t copyDeleteSessionKey(SSHashObj* source, SSHashObj* dest) {

×

3609

  int32_t code = TSDB_CODE_SUCCESS;

×

3610

  int32_t lino = 0;

×

3611

  if (tSimpleHashGetSize(source) == 0) {

×

3612

    goto _end;

×

3614

  void*   pIte = NULL;

×

3615

  int32_t iter = 0;

×

3616

  size_t  keyLen = 0;

×

3617

  while ((pIte = tSimpleHashIterate(source, pIte, &iter)) != NULL) {

×

3618

    SSessionKey* pKey = tSimpleHashGetKey(pIte, &keyLen);

×

3619

    code = saveDeleteRes(dest, *pKey);

×

3620

    QUERY_CHECK_CODE(code, lino, _end);

×

3622

  tSimpleHashClear(source);

×

3624

_end:

×

3625

  if (code != TSDB_CODE_SUCCESS) {

×

3626

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3628

  return code;

×

3631

static int32_t doStreamSessionAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

3632

  int32_t                        code = TSDB_CODE_SUCCESS;

×

3633

  int32_t                        lino = 0;

×

3634

  SExprSupp*                     pSup = &pOperator->exprSupp;

×

3635

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3636

  SOptrBasicInfo*                pBInfo = &pInfo->binfo;

×

3637

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

3638

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

3639

  qDebug("stask:%s  %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);

×

3640

  if (pOperator->status == OP_EXEC_DONE) {

×

3641

    (*ppRes) = NULL;

×

3642

    return code;

×

3643

  } else if (pOperator->status == OP_RES_TO_RETURN) {

×

3644

    SSDataBlock* opRes = NULL;

×

3645

    code = buildSessionResult(pOperator, &opRes);

×

3646

    QUERY_CHECK_CODE(code, lino, _end);

×

3647

    if (opRes) {

×

3648

      (*ppRes) = opRes;

×

3649

      return code;

×

3652

    if (pInfo->recvGetAll) {

×

3653

      pInfo->recvGetAll = false;

×

3654

      resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows);

×

3657

    if (pInfo->reCkBlock) {

×

3658

      pInfo->reCkBlock = false;

×

3659

      printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

3660

      (*ppRes) = pInfo->pCheckpointRes;

×

3661

      return code;

×

3664

    setStreamOperatorCompleted(pOperator);

×

3665

    (*ppRes) = NULL;

×

3666

    return code;

×

3669

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

3670

  if (!pInfo->pUpdated) {

×

3671

    pInfo->pUpdated = taosArrayInit(16, sizeof(SResultWindowInfo));

×

3672

    QUERY_CHECK_NULL(pInfo->pUpdated, code, lino, _end, terrno);

×

3674

  if (!pInfo->pStUpdated) {

×

3675

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

3676

    pInfo->pStUpdated = tSimpleHashInit(64, hashFn);

×

3677

    QUERY_CHECK_NULL(pInfo->pStUpdated, code, lino, _end, terrno);

×

3679

  while (1) {

×

3680

    SSDataBlock* pBlock = NULL;

×

3681

    code = downstream->fpSet.getNextFn(downstream, &pBlock);

×

3682

    QUERY_CHECK_CODE(code, lino, _end);

×

3684

    if (pBlock == NULL) {

×

3685

      break;

×

3687

    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));

×

3688

    setStreamOperatorState(&pInfo->basic, pBlock->info.type);

×

3690

    if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||

×

3691

        pBlock->info.type == STREAM_CLEAR) {

×

3692

      SArray* pWins = taosArrayInit(16, sizeof(SSessionKey));

×

3693

      QUERY_CHECK_NULL(pWins, code, lino, _end, terrno);

×

3695

      code = doDeleteTimeWindows(pAggSup, pBlock, pWins);

×

3696

      QUERY_CHECK_CODE(code, lino, _end);

×

3698

      removeSessionResults(pAggSup, pInfo->pStUpdated, pWins);

×

3699

      if (IS_FINAL_SESSION_OP(pOperator)) {

×

3700

        int32_t                        childIndex = getChildIndex(pBlock);

×

3701

        SOperatorInfo*                 pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);

×

3702

        SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;

×

3704

        code = doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, NULL);

×

3705

        QUERY_CHECK_CODE(code, lino, _end);

×

3707

        code = rebuildSessionWindow(pOperator, pWins, pInfo->pStUpdated);

×

3708

        QUERY_CHECK_CODE(code, lino, _end);

×

3710

      code = copyDeleteWindowInfo(pWins, pInfo->pStDeleted);

×

3711

      QUERY_CHECK_CODE(code, lino, _end);

×

3713

      if (pInfo->destHasPrimaryKey && IS_NORMAL_SESSION_OP(pOperator)) {

×

3714

        code = copyDeleteWindowInfo(pWins, pInfo->pPkDeleted);

×

3715

        QUERY_CHECK_CODE(code, lino, _end);

×

3717

      taosArrayDestroy(pWins);

×

3718

      continue;

×

3719

    } else if (pBlock->info.type == STREAM_GET_ALL) {

×

3720

      pInfo->recvGetAll = true;

×

3721

      code = getAllSessionWindow(pAggSup->pResultRows, pInfo->pStUpdated);

×

3722

      QUERY_CHECK_CODE(code, lino, _end);

×

3723

      continue;

×

3724

    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {

×

3725

      (*ppRes) = pBlock;

×

3726

      return code;

×

3727

    } else if (pBlock->info.type == STREAM_CHECKPOINT) {

×

3728

      pAggSup->stateStore.streamStateCommit(pAggSup->pState);

×

3729

      doStreamSessionSaveCheckpoint(pOperator);

×

3730

      code = copyDataBlock(pInfo->pCheckpointRes, pBlock);

×

3731

      QUERY_CHECK_CODE(code, lino, _end);

×

3733

      continue;

×

3735

      if (pBlock->info.type != STREAM_NORMAL && pBlock->info.type != STREAM_INVALID) {

×

3736

        code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;

×

3737

        QUERY_CHECK_CODE(code, lino, _end);

×

3741

    if (pInfo->scalarSupp.pExprInfo != NULL) {

×

3742

      SExprSupp* pExprSup = &pInfo->scalarSupp;

×

3743

      code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);

×

3744

      QUERY_CHECK_CODE(code, lino, _end);

×

3747

    code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

3748

    QUERY_CHECK_CODE(code, lino, _end);

×

3750

    doStreamSessionAggImpl(pOperator, pBlock, pInfo->pStUpdated, pInfo->pStDeleted, IS_FINAL_SESSION_OP(pOperator),

×

3752

    if (IS_FINAL_SESSION_OP(pOperator)) {

×

3753

      int32_t chIndex = getChildIndex(pBlock);

×

3754

      int32_t size = taosArrayGetSize(pInfo->pChildren);

×

3756

      for (int32_t i = 0; i < chIndex + 1 - size; i++) {

×

3757

        SOperatorInfo* pChildOp = NULL;

×

3758

        code = createStreamFinalSessionAggOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0, NULL, &pChildOp);

×

3759

        if (pChildOp == NULL || code != 0) {

×

3760

          qError("%s create stream child of final session error", GET_TASKID(pTaskInfo));

×

3761

          code = TSDB_CODE_FAILED;

×

3762

          QUERY_CHECK_CODE(code, lino, _end);

×

3765

        void* tmp = taosArrayPush(pInfo->pChildren, &pChildOp);

×

3766

        if (!tmp) {

×

3767

          code = terrno;

×

3768

          QUERY_CHECK_CODE(code, lino, _end);

×

3772

      SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex);

×

3773

      code = setInputDataBlock(&pChildOp->exprSupp, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

3774

      QUERY_CHECK_CODE(code, lino, _end);

×

3775

      doStreamSessionAggImpl(pChildOp, pBlock, NULL, NULL, true, false);

×

3777

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);

×

3778

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.watermark);

×

3781

  pOperator->status = OP_RES_TO_RETURN;

×

3783

  code = closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pInfo->pStUpdated);

×

3784

  QUERY_CHECK_CODE(code, lino, _end);

×

3786

  code = closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs);

×

3787

  QUERY_CHECK_CODE(code, lino, _end);

×

3789

  code = copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc);

×

3790

  QUERY_CHECK_CODE(code, lino, _end);

×

3792

  if (!pInfo->destHasPrimaryKey) {

×

3793

    removeSessionDeleteResults(pInfo->pStDeleted, pInfo->pUpdated);

×

3795

  if (pInfo->isHistoryOp) {

×

3796

    code = getMaxTsWins(pInfo->pUpdated, pInfo->historyWins);

×

3797

    QUERY_CHECK_CODE(code, lino, _end);

×

3799

  if (pInfo->destHasPrimaryKey && IS_NORMAL_SESSION_OP(pOperator)) {

×

3800

    code = copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pStDeleted);

×

3801

    QUERY_CHECK_CODE(code, lino, _end);

×

3803

  initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);

×

3804

  pInfo->pUpdated = NULL;

×

3805

  code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);

×

3806

  QUERY_CHECK_CODE(code, lino, _end);

×

3808

  SSDataBlock* opRes = NULL;

×

3809

  code = buildSessionResult(pOperator, &opRes);

×

3810

  QUERY_CHECK_CODE(code, lino, _end);

×

3811

  if (opRes) {

×

3812

    (*ppRes) = opRes;

×

3813

    return code;

×

3816

_end:

×

3817

  if (code != TSDB_CODE_SUCCESS) {

×

3818

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3819

    pTaskInfo->code = code;

×

3820

    T_LONG_JMP(pTaskInfo->env, code);

×

3822

  setStreamOperatorCompleted(pOperator);

×

3823

  (*ppRes) = NULL;

×

3824

  return code;

×

3827

static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {

×

3828

  SSDataBlock* pRes = NULL;

×

3829

  int32_t      code = doStreamSessionAggNext(pOperator, &pRes);

×

3830

  return pRes;

×

3833

void streamSessionReleaseState(SOperatorInfo* pOperator) {

×

3834

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3835

  int32_t                        winSize = taosArrayGetSize(pInfo->historyWins) * sizeof(SSessionKey);

×

3836

  int32_t                        resSize = winSize + sizeof(TSKEY);

×

3837

  char*                          pBuff = taosMemoryCalloc(1, resSize);

×

3838

  if (!pBuff) {

×

3839

    return;

×

3841

  memcpy(pBuff, pInfo->historyWins->pData, winSize);

×

3842

  memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY));

×

3843

  pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_STATE_NAME,

×

3845

  pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState);

×

3846

  taosMemoryFreeClear(pBuff);

×

3847

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

3848

  if (downstream->fpSet.releaseStreamStateFn) {

×

3849

    downstream->fpSet.releaseStreamStateFn(downstream);

×

3853

void resetWinRange(STimeWindow* winRange) {

×

3854

  winRange->skey = INT64_MIN;

×

3855

  winRange->ekey = INT64_MAX;

×

3856

×

3858

int32_t getSessionWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SResultWindowInfo* pWinInfo) {

×

3859

  int32_t code = TSDB_CODE_SUCCESS;

×

3860

  int32_t lino = 0;

×

3861

  int32_t rowSize = pAggSup->resultRowSize;

×

3862

  int32_t winCode = TSDB_CODE_SUCCESS;

×

3863

  code = pAggSup->stateStore.streamStateSessionGet(pAggSup->pState, pKey, (void**)&pWinInfo->pStatePos, &rowSize,

×

3865

  QUERY_CHECK_CODE(code, lino, _end);

×

3867

  if (winCode == TSDB_CODE_SUCCESS) {

×

3868

    pWinInfo->sessionWin = *pKey;

×

3869

    pWinInfo->isOutput = true;

×

3870

    if (pWinInfo->pStatePos->needFree) {

×

3871

      pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pWinInfo->sessionWin);

×

3874

    SET_SESSION_WIN_INVALID((*pWinInfo));

×

3877

_end:

×

3878

  if (code != TSDB_CODE_SUCCESS) {

×

3879

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3881

  return code;

×

3884

void reloadAggSupFromDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup) {

×

3885

  SStateStore* pAPI = &downstream->pTaskInfo->storageAPI.stateStore;

×

3887

  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {

×

3888

    reloadAggSupFromDownStream(downstream->pDownstream[0], pAggSup);

×

3889

    return;

×

3892

  SStreamScanInfo* pScanInfo = downstream->info;

×

3893

  pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo;

×

3896

void streamSessionSemiReloadState(SOperatorInfo* pOperator) {

×

3897

  int32_t                        code = TSDB_CODE_SUCCESS;

×

3898

  int32_t                        lino = 0;

×

3899

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3900

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

3901

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

3902

  resetWinRange(&pAggSup->winRange);

×

3904

  SResultWindowInfo winInfo = {0};

×

3905

  int32_t           size = 0;

×

3906

  void*             pBuf = NULL;

×

3907

  code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME,

×

3909

  QUERY_CHECK_CODE(code, lino, _end);

×

3911

  int32_t      num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);

×

3912

  SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;

×

3913

  for (int32_t i = 0; i < num; i++) {

×

3914

    SResultWindowInfo winInfo = {0};

×

3915

    code = getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &winInfo);

×

3916

    QUERY_CHECK_CODE(code, lino, _end);

×

3917

    if (!IS_VALID_SESSION_WIN(winInfo)) {

×

3918

      continue;

×

3920

    compactSessionSemiWindow(pOperator, &winInfo);

×

3921

    code = saveSessionOutputBuf(pAggSup, &winInfo);

×

3922

    QUERY_CHECK_CODE(code, lino, _end);

×

3924

  TSKEY ts = *(TSKEY*)((char*)pBuf + size - sizeof(TSKEY));

×

3925

  taosMemoryFree(pBuf);

×

3926

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);

×

3927

  pAggSup->stateStore.streamStateReloadInfo(pAggSup->pState, ts);

×

3929

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

3930

  if (downstream->fpSet.reloadStreamStateFn) {

×

3931

    downstream->fpSet.reloadStreamStateFn(downstream);

×

3933

  reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup);

×

3935

_end:

×

3936

  if (code != TSDB_CODE_SUCCESS) {

×

3937

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3939

×

3941

void streamSessionReloadState(SOperatorInfo* pOperator) {

×

3942

  int32_t                        code = TSDB_CODE_SUCCESS;

×

3943

  int32_t                        lino = 0;

×

3944

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3945

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

3946

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

3947

  resetWinRange(&pAggSup->winRange);

×

3949

  int32_t size = 0;

×

3950

  void*   pBuf = NULL;

×

3951

  code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME,

×

3954

  QUERY_CHECK_CODE(code, lino, _end);

×

3956

  int32_t      num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);

×

3957

  SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;

×

3959

  TSKEY ts = *(TSKEY*)((char*)pBuf + size - sizeof(TSKEY));

×

3960

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);

×

3961

  pAggSup->stateStore.streamStateReloadInfo(pAggSup->pState, ts);

×

3963

  if (!pInfo->pStUpdated && num > 0) {

×

3964

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

3965

    pInfo->pStUpdated = tSimpleHashInit(64, hashFn);

×

3966

    QUERY_CHECK_NULL(pInfo->pStUpdated, code, lino, _end, terrno);

×

3968

  for (int32_t i = 0; i < num; i++) {

×

3969

    SResultWindowInfo winInfo = {0};

×

3970

    code = getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &winInfo);

×

3971

    QUERY_CHECK_CODE(code, lino, _end);

×

3972

    if (!IS_VALID_SESSION_WIN(winInfo)) {

×

3973

      continue;

×

3976

    int32_t winNum = 0;

×

3977

    code = compactSessionWindow(pOperator, &winInfo, pInfo->pStUpdated, pInfo->pStDeleted, true, &winNum);

×

3978

    QUERY_CHECK_CODE(code, lino, _end);

×

3980

    if (winNum > 0) {

×

3981

      qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, winInfo.sessionWin.win.skey,

×

3983

      if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {

×

3984

        code = saveResult(winInfo, pInfo->pStUpdated);

×

3985

        QUERY_CHECK_CODE(code, lino, _end);

×

3986

      } else if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

3987

        if (!isCloseWindow(&winInfo.sessionWin.win, &pInfo->twAggSup)) {

×

3988

          code = saveDeleteRes(pInfo->pStDeleted, winInfo.sessionWin);

×

3989

          QUERY_CHECK_CODE(code, lino, _end);

×

3991

        SSessionKey key = {0};

×

3992

        getSessionHashKey(&winInfo.sessionWin, &key);

×

3993

        code = tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo));

×

3994

        QUERY_CHECK_CODE(code, lino, _end);

×

3997

    code = saveSessionOutputBuf(pAggSup, &winInfo);

×

3998

    QUERY_CHECK_CODE(code, lino, _end);

×

4000

  taosMemoryFree(pBuf);

×

4002

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

4003

  if (downstream->fpSet.reloadStreamStateFn) {

×

4004

    downstream->fpSet.reloadStreamStateFn(downstream);

×

4006

  reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup);

×

4008

_end:

×

4009

  if (code != TSDB_CODE_SUCCESS) {

×

4010

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

4012

×

4014

int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,

×

4016

  QRY_PARAM_CHECK(pOptrInfo);

×

4018

  SSessionWinodwPhysiNode*       pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;

×

4019

  int32_t                        numOfCols = 0;

×

4020

  int32_t                        code = TSDB_CODE_OUT_OF_MEMORY;

×

4021

  int32_t                        lino = 0;

×

4022

  SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo));

×

4023

  SOperatorInfo*                 pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));

×

4024

  if (pInfo == NULL || pOperator == NULL) {

×

4025

    code = terrno;

×

4026

    goto _error;

×

4029

  pOperator->pTaskInfo = pTaskInfo;

×

4031

  initResultSizeInfo(&pOperator->resultInfo, 4096);

×

4032

  if (pSessionNode->window.pExprs != NULL) {

×

4033

    int32_t    numOfScalar = 0;

×

4034

    SExprInfo* pScalarExprInfo = NULL;

×

4035

    code = createExprInfo(pSessionNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalar);

×

4036

    QUERY_CHECK_CODE(code, lino, _error);

×

4038

    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);

×

4039

    if (code != TSDB_CODE_SUCCESS) {

×

4040

      goto _error;

×

4043

  SExprSupp* pExpSup = &pOperator->exprSupp;

×

4045

  SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);

×

4046

  QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);

×

4047

  pInfo->binfo.pRes = pResBlock;

×

4049

  SExprInfo* pExprInfo = NULL;

×

4050

  code = createExprInfo(pSessionNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);

×

4051

  QUERY_CHECK_CODE(code, lino, _error);

×

4053

  code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore);

×

4054

  QUERY_CHECK_CODE(code, lino, _error);

×

4056

  pInfo->twAggSup = (STimeWindowAggSupp){

×

4057

      .waterMark = pSessionNode->window.watermark,

×

4058

      .calTrigger = pSessionNode->window.triggerType,

×

4061

      .deleteMark = getDeleteMark(&pSessionNode->window, 0),

×

4064

  pInfo->primaryTsIndex = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;

×

4066

      initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, pSessionNode->gap, pTaskInfo->streamInfo.pState,

×

4067

                             0, 0, &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup, GET_TASKID(pTaskInfo),

×

4069

  if (code != TSDB_CODE_SUCCESS) {

×

4070

    goto _error;

×

4073

  code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

×

4074

  QUERY_CHECK_CODE(code, lino, _error);

×

4076

  if (pSessionNode->window.pTsEnd) {

×

4077

    pInfo->endTsIndex = ((SColumnNode*)pSessionNode->window.pTsEnd)->slotId;

×

4080

  pInfo->order = TSDB_ORDER_ASC;

×

4081

  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

4082

  pInfo->pStDeleted = tSimpleHashInit(64, hashFn);

×

4083

  QUERY_CHECK_NULL(pInfo->pStDeleted, code, lino, _error, terrno);

×

4084

  pInfo->pDelIterator = NULL;

×

4085

  code = createSpecialDataBlock(STREAM_DELETE_RESULT, &pInfo->pDelRes);

×

4086

  QUERY_CHECK_CODE(code, lino, _error);

×

4088

  pInfo->pChildren = NULL;

×

4089

  pInfo->pPhyNode = pPhyNode;

×

4090

  pInfo->ignoreExpiredData = pSessionNode->window.igExpired;

×

4091

  pInfo->ignoreExpiredDataSaved = false;

×

4092

  pInfo->pUpdated = NULL;

×

4093

  pInfo->pStUpdated = NULL;

×

4094

  pInfo->dataVersion = 0;

×

4095

  pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey));

×

4096

  if (!pInfo->historyWins) {

×

4097

    goto _error;

×

4099

  if (pHandle) {

×

4100

    pInfo->isHistoryOp = pHandle->fillHistory;

×

4103

  code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);

×

4104

  QUERY_CHECK_CODE(code, lino, _error);

×

4106

  pInfo->clearState = false;

×

4107

  pInfo->recvGetAll = false;

×

4108

  pInfo->destHasPrimaryKey = pSessionNode->window.destHasPrimaryKey;

×

4109

  pInfo->pPkDeleted = tSimpleHashInit(64, hashFn);

×

4110

  QUERY_CHECK_NULL(pInfo->pPkDeleted, code, lino, _error, terrno);

×

4111

  pInfo->pOperator = pOperator;

×

4113

  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;

×

4114

  setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true,

×

4116

  if (pPhyNode->type != QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {

×

4118

    void*   buff = NULL;

×

4119

    int32_t len = 0;

×

4121

        pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME,

×

4123

    if (res == TSDB_CODE_SUCCESS) {

×

4124

      code = doStreamSessionDecodeOpState(buff, len, pOperator, true, NULL);

×

4125

      taosMemoryFree(buff);

×

4126

      QUERY_CHECK_CODE(code, lino, _error);

×

4130

      createOperatorFpSet(optrDummyOpenFn, doStreamSessionAggNext, NULL, destroyStreamSessionAggOperatorInfo,

×

4132

  setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState);

×

4134

  code = initStreamBasicInfo(&pInfo->basic, pOperator);

×

4135

  QUERY_CHECK_CODE(code, lino, _error);

×

4137

  if (downstream) {

×

4138

    pInfo->basic.primaryPkIndex = -1;

×

4139

    code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,

×

4140

                          &pInfo->twAggSup, &pInfo->basic);

×

4141

    QUERY_CHECK_CODE(code, lino, _error);

×

4143

    code = appendDownstream(pOperator, &downstream, 1);

×

4144

    QUERY_CHECK_CODE(code, lino, _error);

×

4147

  *pOptrInfo = pOperator;

×

4148

  return TSDB_CODE_SUCCESS;

×

4150

_error:

×

4151

  if (pInfo != NULL) {

×

4152

    destroyStreamSessionAggOperatorInfo(pInfo);

×

4154

  destroyOperatorAndDownstreams(pOperator, &downstream, 1);

×

4155

  pTaskInfo->code = code;

×

4156

  qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

4157

  return code;

×

4160

static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) {

×

4161

  tSimpleHashClear(pInfo->streamAggSup.pResultRows);

×

4162

  pInfo->streamAggSup.stateStore.streamStateSessionClear(pInfo->streamAggSup.pState);

×

4163

  pInfo->clearState = false;

×

4164

×

4166

int32_t deleteSessionWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate,

×

4168

  int32_t code = TSDB_CODE_SUCCESS;

×

4169

  int32_t lino = 0;

×

4170

  SArray* pWins = taosArrayInit(16, sizeof(SSessionKey));

×

4171

  if (!pWins) {

×

4172

    code = terrno;

×

4173

    QUERY_CHECK_CODE(code, lino, _end);

×

4175

  code = doDeleteTimeWindows(pAggSup, pBlock, pWins);

×

4176

  QUERY_CHECK_CODE(code, lino, _end);

×

4178

  removeSessionResults(pAggSup, pMapUpdate, pWins);

×

4179

  code = copyDeleteWindowInfo(pWins, pMapDelete);

×

4180

  QUERY_CHECK_CODE(code, lino, _end);

×

4182

  if (needAdd) {

×

4183

    code = copyDeleteWindowInfo(pWins, pPkDelete);

×

4184

    QUERY_CHECK_CODE(code, lino, _end);

×

4186

  taosArrayDestroy(pWins);

×

4188

_end:

×

4189

  if (code != TSDB_CODE_SUCCESS) {

×

4190

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

4192

  return code;

×

4195

static int32_t doStreamSessionSemiAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

4196

  int32_t                        code = TSDB_CODE_SUCCESS;

×

4197

  int32_t                        lino = 0;

×

4198

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

4199

  SOptrBasicInfo*                pBInfo = &pInfo->binfo;

×

4200

  TSKEY                          maxTs = INT64_MIN;

×

4201

  SExprSupp*                     pSup = &pOperator->exprSupp;

×

4202

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

4203

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

4205

  qDebug("stask:%s  %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);

×

4206

  if (pOperator->status == OP_EXEC_DONE) {

×

4207

    (*ppRes) = NULL;

×

4208

    return code;

×

4212

    SSDataBlock* opRes = NULL;

×

4213

    code = buildSessionResult(pOperator, &opRes);

×

4214

    QUERY_CHECK_CODE(code, lino, _end);

×

4215

    if (opRes) {

×

4216

      (*ppRes) = opRes;

×

4217

      return code;

×

4220

    if (pInfo->clearState) {

×

4221

      clearFunctionContext(&pOperator->exprSupp);

×

4223

      clearStreamSessionOperator(pInfo);

×

4226

    if (pOperator->status == OP_RES_TO_RETURN) {

×

4227

      if (pInfo->reCkBlock) {

×

4228

        pInfo->reCkBlock = false;

×

4229

        printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

4230

        (*ppRes) = pInfo->pCheckpointRes;

×

4231

        return code;

×

4233

      clearFunctionContext(&pOperator->exprSupp);

×

4235

      clearStreamSessionOperator(pInfo);

×

4236

      setStreamOperatorCompleted(pOperator);

×

4237

      (*ppRes) = NULL;

×

4238

      return code;

×

4242

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

4243

  if (!pInfo->pUpdated) {

×

4244

    pInfo->pUpdated = taosArrayInit(16, sizeof(SResultWindowInfo));

×

4245

    QUERY_CHECK_NULL(pInfo->pUpdated, code, lino, _end, terrno);

×

4247

  if (!pInfo->pStUpdated) {

×

4248

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

4249

    pInfo->pStUpdated = tSimpleHashInit(64, hashFn);

×

4250

    QUERY_CHECK_NULL(pInfo->pStUpdated, code, lino, _end, terrno);

×

4252

  while (1) {

×

4253

    SSDataBlock* pBlock = NULL;

×

4254

    code = downstream->fpSet.getNextFn(downstream, &pBlock);

×

4255

    QUERY_CHECK_CODE(code, lino, _end);

×

4257

    if (pBlock == NULL) {

×

4258

      pOperator->status = OP_RES_TO_RETURN;

×

4259

      break;

×

4261

    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));

×

4262

    setStreamOperatorState(&pInfo->basic, pBlock->info.type);

×

4264

    if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||

×

4265

        pBlock->info.type == STREAM_CLEAR) {

×

4267

      code = deleteSessionWinState(pAggSup, pBlock, pInfo->pStUpdated, pInfo->pStDeleted, NULL, false);

×

4268

      QUERY_CHECK_CODE(code, lino, _end);

×

4269

      pInfo->clearState = true;

×

4270

      break;

×

4271

    } else if (pBlock->info.type == STREAM_GET_ALL) {

×

4272

      code = getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pStUpdated);

×

4273

      QUERY_CHECK_CODE(code, lino, _end);

×

4274

      continue;

×

4275

    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {

×

4276

      (*ppRes) = pBlock;

×

4277

      return code;

×

4278

    } else if (pBlock->info.type == STREAM_CHECKPOINT) {

×

4279

      pAggSup->stateStore.streamStateCommit(pAggSup->pState);

×

4280

      doStreamSessionSaveCheckpoint(pOperator);

×

4281

      continue;

×

4283

      if (pBlock->info.type != STREAM_NORMAL && pBlock->info.type != STREAM_INVALID) {

×

4284

        code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;

×

4285

        QUERY_CHECK_CODE(code, lino, _end);

×

4289

    if (pInfo->scalarSupp.pExprInfo != NULL) {

×

4290

      SExprSupp* pExprSup = &pInfo->scalarSupp;

×

4291

      code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);

×

4292

      QUERY_CHECK_CODE(code, lino, _end);

×

4295

    code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

4296

    QUERY_CHECK_CODE(code, lino, _end);

×

4297

    doStreamSessionAggImpl(pOperator, pBlock, pInfo->pStUpdated, NULL, false, false);

×

4298

    maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);

×

4301

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);

×

4302

  pBInfo->pRes->info.watermark = pInfo->twAggSup.maxTs;

×

4304

  code = copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc);

×

4305

  QUERY_CHECK_CODE(code, lino, _end);

×

4307

  removeSessionDeleteResults(pInfo->pStDeleted, pInfo->pUpdated);

×

4309

  if (pInfo->isHistoryOp) {

×

4310

    code = getMaxTsWins(pInfo->pUpdated, pInfo->historyWins);

×

4311

    QUERY_CHECK_CODE(code, lino, _end);

×

4314

  initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);

×

4315

  pInfo->pUpdated = NULL;

×

4316

  code = blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);

×

4317

  QUERY_CHECK_CODE(code, lino, _end);

×

4319

  SSDataBlock* opRes = NULL;

×

4320

  code = buildSessionResult(pOperator, &opRes);

×

4321

  QUERY_CHECK_CODE(code, lino, _end);

×

4322

  if (opRes) {

×

4323

    (*ppRes) = opRes;

×

4324

    return code;

×

4327

_end:

×

4328

  if (code != TSDB_CODE_SUCCESS) {

×

4329

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

4330

    pTaskInfo->code = code;

×

4331

    T_LONG_JMP(pTaskInfo->env, code);

×

4334

  clearFunctionContext(&pOperator->exprSupp);

×

4336

  clearStreamSessionOperator(pInfo);

×

4337

  setStreamOperatorCompleted(pOperator);

×

4338

  (*ppRes) = NULL;

×

4339

  return code;

×

4342

static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {

×

4343

  SSDataBlock* pRes = NULL;

×

4344

  int32_t      code = doStreamSessionSemiAggNext(pOperator, &pRes);

×

4345

  return pRes;

×

4348

int32_t createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,

×

4351

  QRY_PARAM_CHECK(pOptrInfo);

×

4353

  int32_t        code = TSDB_CODE_SUCCESS;

×

4354

  int32_t        lino = 0;

×

4355

  SOperatorInfo* pOperator = NULL;

×

4356

  code = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo, pHandle, &pOperator);

×

4357

  if (pOperator == NULL || code != 0) {

×

4358

    downstream = NULL;

×

4359

    QUERY_CHECK_CODE(code, lino, _error);

×

4362

  SStorageAPI*                   pAPI = &pTaskInfo->storageAPI;

×

4363

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

4364

  pOperator->operatorType = pPhyNode->type;

×

4365

  pInfo->pOperator = pOperator;

×

4367

  if (pPhyNode->type != QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {

×

4368

    pOperator->fpSet =

×

4369

        createOperatorFpSet(optrDummyOpenFn, doStreamSessionSemiAggNext, NULL, destroyStreamSessionAggOperatorInfo,

×

4371

    setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionSemiReloadState);

×

4374

  code = initStreamBasicInfo(&pInfo->basic, pOperator);

×

4375

  QUERY_CHECK_CODE(code, lino, _error);

×

4377

  setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), pPhyNode->type, false, OP_NOT_OPENED, pInfo,

×

4380

  if (numOfChild > 0) {

×

4381

    pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*));

×

4382

    QUERY_CHECK_NULL(pInfo->pChildren, code, lino, _error, terrno);

×

4383

    for (int32_t i = 0; i < numOfChild; i++) {

×

4384

      SOperatorInfo* pChildOp = NULL;

×

4385

      code = createStreamFinalSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo, 0, pHandle, &pChildOp);

×

4386

      if (pChildOp == NULL || code != 0) {

×

4387

        QUERY_CHECK_CODE(code, lino, _error);

×

4390

      SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info;

×

4391

      pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;

×

4392

      pAPI->stateStore.streamStateSetNumber(pChInfo->streamAggSup.pState, i, pInfo->primaryTsIndex);

×

4393

      void* tmp = taosArrayPush(pInfo->pChildren, &pChildOp);

×

4394

      if (!tmp) {

×

4395

        code = terrno;

×

4396

        QUERY_CHECK_CODE(code, lino, _error);

×

4400

    void*   buff = NULL;

×

4401

    int32_t len = 0;

×

4403

        pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME,

×

4405

    if (res == TSDB_CODE_SUCCESS) {

×

4406

      code = doStreamSessionDecodeOpState(buff, len, pOperator, true, NULL);

×

4407

      taosMemoryFree(buff);

×

4408

      QUERY_CHECK_CODE(code, lino, _error);

×

4412

  if (!IS_FINAL_SESSION_OP(pOperator) || numOfChild == 0) {

×

4413

    pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;

×

4416

  *pOptrInfo = pOperator;

×

4417

  return code;

×

4419

_error:

×

4420

  if (pInfo != NULL) {

×

4421

    destroyStreamSessionAggOperatorInfo(pInfo);

×

4423

  if (pOperator != NULL) {

×

4424

    pOperator->info = NULL;

×

4425

    destroyOperator(pOperator);

×

4427

  pTaskInfo->code = code;

×

4428

  if (code != TSDB_CODE_SUCCESS) {

×

4429

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

4431

  return code;

×

4434

void destroyStreamStateOperatorInfo(void* param) {

×

4435

  if (param == NULL) {

×

4436

    return;

×

4438

  SStreamStateAggOperatorInfo* pInfo = (SStreamStateAggOperatorInfo*)param;

×

4439

  cleanupBasicInfo(&pInfo->binfo);

×

4440

  if (pInfo->pOperator) {

×

4441

    cleanupResultInfoInStream(pInfo->pOperator->pTaskInfo, pInfo->streamAggSup.pState, &pInfo->pOperator->exprSupp,

×

4443

    pInfo->pOperator = NULL;

×

4446

  destroyStreamBasicInfo(&pInfo->basic);

×

4447

  clearGroupResInfo(&pInfo->groupResInfo);

×

4448

  taosArrayDestroyP(pInfo->pUpdated, destroyFlusedPos);

×

4449

  pInfo->pUpdated = NULL;

×

4450

  destroyStreamAggSupporter(&pInfo->streamAggSup);

×

4452

  cleanupExprSupp(&pInfo->scalarSupp);

×

4453

  if (pInfo->pChildren != NULL) {

×

4454

    int32_t size = taosArrayGetSize(pInfo->pChildren);

×

4455

    for (int32_t i = 0; i < size; i++) {

×

4456

      SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, i);

×

4457

      destroyOperator(pChild);

×

4459

    taosArrayDestroy(pInfo->pChildren);

×

4461

  colDataDestroy(&pInfo->twAggSup.timeWindowData);

×

4462

  blockDataDestroy(pInfo->pDelRes);

×

4463

  tSimpleHashCleanup(pInfo->pSeUpdated);

×

4464

  tSimpleHashCleanup(pInfo->pSeDeleted);

×

4465

  cleanupGroupResInfo(&pInfo->groupResInfo);

×

4467

  taosArrayDestroy(pInfo->historyWins);

×

4468

  blockDataDestroy(pInfo->pCheckpointRes);

×

4469

  tSimpleHashCleanup(pInfo->pPkDeleted);

×

4471

  taosMemoryFreeClear(param);

×

4474

bool isTsInWindow(SStateWindowInfo* pWin, TSKEY ts) {

×

4475

  if (pWin->winInfo.sessionWin.win.skey <= ts && ts <= pWin->winInfo.sessionWin.win.ekey) {

×

4476

    return true;

×

4478

  return false;

×

4481

bool isEqualStateKey(SStateWindowInfo* pWin, char* pKeyData) {

×

4482

  return pKeyData && compareVal(pKeyData, pWin->pStateKey);

×

4485

bool compareStateKey(void* data, void* key) {

×

4486

  if (!data || !key) {

×

4487

    return true;

×

4489

  SStateKeys* stateKey = (SStateKeys*)key;

×

4490

  stateKey->pData = (char*)key + sizeof(SStateKeys);

×

4491

  return compareVal(data, stateKey);

×

4494

bool compareWinStateKey(SStateKeys* left, SStateKeys* right) {

×

4495

  if (!left || !right) {

×

4496

    return false;

×

4498

  return compareVal(left->pData, right);

×

4501

static void getNextStateWin(const SStreamAggSupporter* pAggSup, SStateWindowInfo* pNextWin, bool asc) {

×

4502

  SStreamStateCur* pCur = NULL;

×

4504

  if (pAggSup == NULL || pNextWin == NULL) {

×

4505

    return;

×

4508

  if (asc)

×

4509

    pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pNextWin->winInfo.sessionWin);

×

4511

    pCur = pAggSup->stateStore.streamStateSessionSeekKeyPrev(pAggSup->pState, &pNextWin->winInfo.sessionWin);

×

4512

  int32_t nextSize = pAggSup->resultRowSize;

×

4513

  int32_t winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin,

×

4514

                                                                     (void**)&pNextWin->winInfo.pStatePos, &nextSize);

×

4515

  if (winCode != TSDB_CODE_SUCCESS) {

×

4516

    SET_SESSION_WIN_INVALID(pNextWin->winInfo);

×

4518

    pNextWin->pStateKey =

×

4519

        (SStateKeys*)((char*)pNextWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));

×

4520

    pNextWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);

×

4521

    pNextWin->pStateKey->type = pAggSup->stateKeyType;

×

4522

    pNextWin->pStateKey->pData = (char*)pNextWin->pStateKey + sizeof(SStateKeys);

×

4523

    pNextWin->pStateKey->isNull = false;

×

4524

    pNextWin->winInfo.isOutput = true;

×

4526

  pAggSup->stateStore.streamStateFreeCur(pCur);

×

4529

int32_t getStateWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SStateWindowInfo* pCurWin,

×

4531

  int32_t code = TSDB_CODE_SUCCESS;

×

4532

  int32_t lino = 0;

×

4533

  int32_t size = pAggSup->resultRowSize;

×

4534

  pCurWin->winInfo.sessionWin.groupId = pKey->groupId;

×

4535

  pCurWin->winInfo.sessionWin.win.skey = pKey->win.skey;

×

4536

  pCurWin->winInfo.sessionWin.win.ekey = pKey->win.ekey;

×

4537

  code = getSessionWindowInfoByKey(pAggSup, pKey, &pCurWin->winInfo);

×

4538

  QUERY_CHECK_CODE(code, lino, _end);

×

4539

  QUERY_CHECK_CONDITION((IS_VALID_SESSION_WIN(pCurWin->winInfo)), code, lino, _end,

×

4542

  pCurWin->pStateKey =

×

4543

      (SStateKeys*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));

×

4544

  pCurWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);

×

4545

  pCurWin->pStateKey->type = pAggSup->stateKeyType;

×

4546

  pCurWin->pStateKey->pData = (char*)pCurWin->pStateKey + sizeof(SStateKeys);

×

4547

  pCurWin->pStateKey->isNull = false;

×

4548

  pCurWin->winInfo.isOutput = true;

×

4549

  if (pCurWin->winInfo.pStatePos->needFree) {

×

4550

    pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->winInfo.sessionWin);

×

4553

  qDebug("===stream===get state cur win buff. skey:%" PRId64 ", endkey:%" PRId64, pCurWin->winInfo.sessionWin.win.skey,

×

4556

  pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin;

×

4557

  getNextStateWin(pAggSup, pNextWin, true);

×

4559

_end:

×

4560

  qDebug("===stream===get state next win buff. skey:%" PRId64 ", endkey:%" PRId64,

×

4562

  if (code != TSDB_CODE_SUCCESS) {

×

4563

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

4565

  return code;

×

4568

int32_t setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData,

×

4570

  int32_t size = pAggSup->resultRowSize;

×

4571

  pCurWin->winInfo.sessionWin.groupId = groupId;

×

4572

  pCurWin->winInfo.sessionWin.win.skey = ts;

×

4573

  pCurWin->winInfo.sessionWin.win.ekey = ts;

×

4574

  int32_t code = TSDB_CODE_SUCCESS;

×

4575

  int32_t lino = 0;

×

4576

  int32_t winCode = TSDB_CODE_SUCCESS;

×

4577

  code = pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, pKeyData,

×

4579

                                                           (void**)&pCurWin->winInfo.pStatePos, &size, &winCode);

×

4580

  QUERY_CHECK_CODE(code, lino, _end);

×

4582

  pCurWin->pStateKey =

×

4583

      (SStateKeys*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));

×

4584

  pCurWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);

×

4585

  pCurWin->pStateKey->type = pAggSup->stateKeyType;

×

4586

  pCurWin->pStateKey->pData = (char*)pCurWin->pStateKey + sizeof(SStateKeys);

×

4587

  pCurWin->pStateKey->isNull = false;

×

4589

  if (winCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->winInfo.sessionWin.win)) {

×

4590

    winCode = TSDB_CODE_FAILED;

×

4591

    clearOutputBuf(pAggSup->pState, pCurWin->winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);

×

4592

    pCurWin->pStateKey =

×

4593

        (SStateKeys*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));

×

4594

    pCurWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);

×

4595

    pCurWin->pStateKey->type = pAggSup->stateKeyType;

×

4596

    pCurWin->pStateKey->pData = (char*)pCurWin->pStateKey + sizeof(SStateKeys);

×

4597

    pCurWin->pStateKey->isNull = false;

×

4598

    pCurWin->winInfo.sessionWin.groupId = groupId;

×

4599

    pCurWin->winInfo.sessionWin.win.skey = ts;

×

4600

    pCurWin->winInfo.sessionWin.win.ekey = ts;

×

4601

    qDebug("===stream===reset state win key. skey:%" PRId64 ", endkey:%" PRId64, pCurWin->winInfo.sessionWin.win.skey,

×

4605

  if (winCode == TSDB_CODE_SUCCESS) {

×

4606

    pCurWin->winInfo.isOutput = true;

×

4607

    if (pCurWin->winInfo.pStatePos->needFree) {

×

4608

      pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->winInfo.sessionWin);

×

4610

  } else if (pKeyData) {

×

4611

    if (IS_VAR_DATA_TYPE(pAggSup->stateKeyType)) {

×

4612

      varDataCopy(pCurWin->pStateKey->pData, pKeyData);

×

4614

      memcpy(pCurWin->pStateKey->pData, pKeyData, pCurWin->pStateKey->bytes);

×

4618

  *pWinCode = winCode;

×

4620

  qDebug("===stream===set state cur win buff. skey:%" PRId64 ", endkey:%" PRId64, pCurWin->winInfo.sessionWin.win.skey,

×

4623

  pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin;

×

4624

  getNextStateWin(pAggSup, pNextWin, true);

×

4625

  qDebug("===stream===set state next win buff. skey:%" PRId64 ", endkey:%" PRId64,

×

4627

_end:

×

4628

  if (code != TSDB_CODE_SUCCESS) {

×

4629

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

4631

  return code;

×

4634

int32_t updateStateWindowInfo(SStreamAggSupporter* pAggSup, SStateWindowInfo* pWinInfo, SStateWindowInfo* pNextWin,

×

4638

  int32_t code = TSDB_CODE_SUCCESS;

×

4639

  int32_t lino = 0;

×

4640

  *allEqual = true;

×

4641

  for (int32_t i = start; i < rows; ++i) {

×

4642

    char* pKeyData = colDataGetData(pKeyCol, i);

×

4643

    if (!isTsInWindow(pWinInfo, pTs[i])) {

×

4644

      if (isEqualStateKey(pWinInfo, pKeyData)) {

×

4645

        if (IS_VALID_SESSION_WIN(pNextWin->winInfo)) {

×

4647

          if (pTs[i] >= pNextWin->winInfo.sessionWin.win.skey) {

×

4648

            (*pWinRows) = i - start;

×

4649

            goto _end;

×

4653

        (*pWinRows) = i - start;

×

4654

        goto _end;

×

4658

    if (pWinInfo->winInfo.sessionWin.win.skey > pTs[i]) {

×

4659

      if (pSeDeleted && pWinInfo->winInfo.isOutput) {

×

4660

        code = saveDeleteRes(pSeDeleted, pWinInfo->winInfo.sessionWin);

×

4661

        QUERY_CHECK_CODE(code, lino, _end);

×

4663

      removeSessionResult(pAggSup, pSeUpdated, pResultRows, &pWinInfo->winInfo.sessionWin);

×

4664

      pWinInfo->winInfo.sessionWin.win.skey = pTs[i];

×

4666

    pWinInfo->winInfo.sessionWin.win.ekey = TMAX(pWinInfo->winInfo.sessionWin.win.ekey, pTs[i]);

×

4667

    memcpy(pWinInfo->winInfo.pStatePos->pKey, &pWinInfo->winInfo.sessionWin, sizeof(SSessionKey));

×

4668

    if (!isEqualStateKey(pWinInfo, pKeyData)) {

×

4669

      *allEqual = false;

×

4672

  (*pWinRows) = rows - start;

×

4674

_end:

×

4675

  if (code != TSDB_CODE_SUCCESS) {

×

4676

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

4678

  return code;

×

4681

static bool isWinResult(SSessionKey* pKey, SSHashObj* pSeUpdate, SSHashObj* pResults) {

×

4682

  SSessionKey checkKey = {0};

×

4683

  getSessionHashKey(pKey, &checkKey);

×

4684

  if (tSimpleHashGet(pSeUpdate, &checkKey, sizeof(SSessionKey)) != NULL) {

×

4685

    return true;

×

4688

  if (tSimpleHashGet(pResults, &checkKey, sizeof(SSessionKey)) != NULL) {

×

4689

    return true;

×

4691

  return false;

×

4694

static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pSeUpdated,

×

4696

  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

×

4697

  SStorageAPI*   pAPI = &pOperator->pTaskInfo->storageAPI;

×

4699

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

4700

  int32_t                      numOfOutput = pOperator->exprSupp.numOfExprs;

×

4701

  uint64_t                     groupId = pSDataBlock->info.id.groupId;

×

4702

  int32_t                      code = TSDB_CODE_SUCCESS;

×

4703

  int32_t                      lino = 0;

×

4704

  TSKEY*                       tsCols = NULL;

×

4705

  SResultRow*                  pResult = NULL;

×

4706

  int32_t                      winRows = 0;

×

4707

  SStreamAggSupporter*         pAggSup = &pInfo->streamAggSup;

×

4708

  SStreamNotifyEventSupp*      pNotifySup = &pInfo->basic.notifyEventSup;

×

4709

  STaskNotifyEventStat*        pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;

×

4711

  pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);

×

4712

  pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;

×

4713

  if (pAggSup->winRange.ekey <= 0) {

×

4714

    pAggSup->winRange.ekey = INT64_MAX;

×

4717

  if (pSDataBlock->pDataBlock != NULL) {

×

4718

    SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);

×

4719

    if (!pColDataInfo) {

×

4720

      code = TSDB_CODE_FAILED;

×

4721

      QUERY_CHECK_CODE(code, lino, _end);

×

4723

    tsCols = (int64_t*)pColDataInfo->pData;

×

4725

    return;

×

4728

  int32_t rows = pSDataBlock->info.rows;

×

4729

  code = blockDataEnsureCapacity(pAggSup->pScanBlock, rows);

×

4730

  QUERY_CHECK_CODE(code, lino, _end);

×

4732

  SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);

×

4733

  for (int32_t i = 0; i < rows; i += winRows) {

×

4734

    if (pInfo->ignoreExpiredData && checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo,

×

4735

                                                     &pInfo->twAggSup, pSDataBlock->info.id.uid, tsCols[i], NULL, 0) ||

×

4736

        colDataIsNull_s(pKeyColInfo, i)) {

×

4737

      i++;

×

4738

      continue;

×

4740

    char*            pKeyData = colDataGetData(pKeyColInfo, i);

×

4741

    int32_t          winIndex = 0;

×

4742

    bool             allEqual = true;

×

4743

    SStateWindowInfo curWin = {0};

×

4744

    SStateWindowInfo nextWin = {0};

×

4745

    int32_t          winCode = TSDB_CODE_SUCCESS;

×

4746

    code = setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin, &winCode);

×

4747

    QUERY_CHECK_CODE(code, lino, _end);

×

4749

    if (winCode != TSDB_CODE_SUCCESS && pTaskInfo->streamInfo.eventTypes) {

×

4750

      SStateWindowInfo prevWin = {.winInfo.sessionWin = curWin.winInfo.sessionWin};

×

4751

      getNextStateWin(pAggSup, &prevWin, false);

×

4752

      qDebug("===stream===get state prev win buff. skey:%" PRId64 ", endkey:%" PRId64,

×

4754

      releaseOutputBuf(pAggSup->pState, prevWin.winInfo.pStatePos, &pAPI->stateStore);

×

4757

      if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE) &&

×

4758

          IS_VALID_SESSION_WIN(prevWin.winInfo)) {

×

4759

        code = addStateAggNotifyEvent(SNOTIFY_EVENT_WINDOW_CLOSE, &prevWin.winInfo.sessionWin, prevWin.pStateKey,

×

4760

                                      curWin.pStateKey, true, pNotifySup, pNotifyEventStat);

×

4761

        QUERY_CHECK_CODE(code, lino, _end);

×

4763

      if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {

×

4764

        code = addStateAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &curWin.winInfo.sessionWin, curWin.pStateKey,

×

4765

                                      prevWin.pStateKey, false, pNotifySup, pNotifyEventStat);

×

4766

        QUERY_CHECK_CODE(code, lino, _end);

×

4770

    if (isWinResult(&nextWin.winInfo.sessionWin, pSeUpdated, pAggSup->pResultRows) == false) {

×

4771

      releaseOutputBuf(pAggSup->pState, nextWin.winInfo.pStatePos, &pAPI->stateStore);

×

4774

    setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo);

×

4775

    code = updateStateWindowInfo(pAggSup, &curWin, &nextWin, tsCols, groupId, pKeyColInfo, rows, i, &allEqual,

×

4777

    QUERY_CHECK_CODE(code, lino, _end);

×

4779

    if (!allEqual) {

×

4780

      uint64_t uid = 0;

×

4781

      code = appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey,

×

4783

      QUERY_CHECK_CODE(code, lino, _end);

×

4784

      int32_t tmpRes = tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey));

×

4785

      qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

4787

      doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin);

×

4788

      releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore);

×

4789

      continue;

×

4792

    code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput,

×

4794

    QUERY_CHECK_CODE(code, lino, _end);

×

4796

    code = saveSessionOutputBuf(pAggSup, &curWin.winInfo);

×

4797

    QUERY_CHECK_CODE(code, lino, _end);

×

4799

    if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_STATE_OP(pOperator)) {

×

4800

      code = saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin);

×

4801

      QUERY_CHECK_CODE(code, lino, _end);

×

4804

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {

×

4805

      code = saveResult(curWin.winInfo, pSeUpdated);

×

4806

      QUERY_CHECK_CODE(code, lino, _end);

×

4809

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

4810

      curWin.winInfo.pStatePos->beUpdated = true;

×

4811

      SSessionKey key = {0};

×

4812

      getSessionHashKey(&curWin.winInfo.sessionWin, &key);

×

4814

          tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo));

×

4815

      QUERY_CHECK_CODE(code, lino, _end);

×

4820

    if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE)) {

×

4821

      code = addStateAggNotifyEvent(SNOTIFY_EVENT_WINDOW_CLOSE, &curWin.winInfo.sessionWin, curWin.pStateKey,

×

4822

                                    nextWin.pStateKey, false, pNotifySup, pNotifyEventStat);

×

4823

      QUERY_CHECK_CODE(code, lino, _end);

×

4827

_end:

×

4828

  if (code != TSDB_CODE_SUCCESS) {

×

4829

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

4833

int32_t doStreamStateEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOperator, bool isParent) {

×

4834

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

4835

  if (!pInfo) {

×

4836

    return 0;

×

4839

  void* pData = (buf == NULL) ? NULL : *buf;

×

4842

  int32_t tlen = 0;

×

4843

  int32_t mapSize = tSimpleHashGetSize(pInfo->streamAggSup.pResultRows);

×

4844

  tlen += taosEncodeFixedI32(buf, mapSize);

×

4845

  void*   pIte = NULL;

×

4846

  size_t  keyLen = 0;

×

4847

  int32_t iter = 0;

×

4848

  while ((pIte = tSimpleHashIterate(pInfo->streamAggSup.pResultRows, pIte, &iter)) != NULL) {

×

4849

    void* key = tSimpleHashGetKey(pIte, &keyLen);

×

4850

    tlen += encodeSSessionKey(buf, key);

×

4851

    tlen += encodeSResultWindowInfo(buf, pIte, pInfo->streamAggSup.resultRowSize);

×

4855

  tlen += encodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);

×

4858

  int32_t size = taosArrayGetSize(pInfo->pChildren);

×

4859

  tlen += taosEncodeFixedI32(buf, size);

×

4860

  for (int32_t i = 0; i < size; i++) {

×

4861

    SOperatorInfo* pChOp = taosArrayGetP(pInfo->pChildren, i);

×

4862

    tlen += doStreamStateEncodeOpState(buf, 0, pChOp, false);

×

4866

  tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);

×

4869

  tlen += encodeStreamBasicInfo(buf, &pInfo->basic);

×

4872

  if (isParent) {

×

4873

    if (buf) {

×

4874

      uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));

×

4875

      tlen += taosEncodeFixedU32(buf, cksum);

×

4877

      tlen += sizeof(uint32_t);

×

4881

  return tlen;

×

4884

int32_t doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator, bool isParent, void** ppBuf) {

×

4885

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

4886

  int32_t                      code = TSDB_CODE_SUCCESS;

×

4887

  int32_t                      lino = 0;

×

4888

  SStreamAggSupporter*         pAggSup = &pInfo->streamAggSup;

×

4889

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

4890

  void*                        pDataEnd = POINTER_SHIFT(buf, len);

×

4891

  if (!pInfo) {

×

4892

    code = TSDB_CODE_FAILED;

×

4893

    QUERY_CHECK_CODE(code, lino, _end);

×

4897

  if (isParent) {

×

4898

    int32_t dataLen = len - sizeof(uint32_t);

×

4899

    void*   pCksum = POINTER_SHIFT(buf, dataLen);

×

4900

    if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) {

×

4901

      qError("stream state_window state is invalid");

×

4902

      code = TSDB_CODE_FAILED;

×

4903

      QUERY_CHECK_CODE(code, lino, _end);

×

4905

    pDataEnd = pCksum;

×

4909

  int32_t mapSize = 0;

×

4910

  buf = taosDecodeFixedI32(buf, &mapSize);

×

4911

  for (int32_t i = 0; i < mapSize; i++) {

×

4912

    SResultWindowInfo winfo = {0};

×

4913

    buf = decodeSSessionKey(buf, &winfo.sessionWin);

×

4914

    int32_t winCode = TSDB_CODE_SUCCESS;

×

4915

    code = pAggSup->stateStore.streamStateStateAddIfNotExist(

×

4918

    QUERY_CHECK_CODE(code, lino, _end);

×

4920

    buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize);

×

4921

    code = tSimpleHashPut(pInfo->streamAggSup.pResultRows, &winfo.sessionWin, sizeof(SSessionKey), &winfo,

×

4923

    QUERY_CHECK_CODE(code, lino, _end);

×

4927

  buf = decodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);

×

4930

  int32_t size = 0;

×

4931

  buf = taosDecodeFixedI32(buf, &size);

×

4932

  for (int32_t i = 0; i < size; i++) {

×

4933

    SOperatorInfo* pChOp = taosArrayGetP(pInfo->pChildren, i);

×

4934

    code = doStreamStateDecodeOpState(buf, 0, pChOp, false, &buf);

×

4935

    QUERY_CHECK_CODE(code, lino, _end);

×

4939

  buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);

×

4941

  if (ppBuf) {

×

4942

    (*ppBuf) = buf;

×

4946

  if (buf < pDataEnd) {

×

4947

    code = decodeStreamBasicInfo(&buf, &pInfo->basic);

×

4948

    QUERY_CHECK_CODE(code, lino, _end);

×

4951

_end:

×

4952

  if (code != TSDB_CODE_SUCCESS) {

×

4953

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

4955

  return code;

×

4958

void doStreamStateSaveCheckpoint(SOperatorInfo* pOperator) {

×

4959

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

4960

  if (needSaveStreamOperatorInfo(&pInfo->basic)) {

×

4961

    int32_t len = doStreamStateEncodeOpState(NULL, 0, pOperator, true);

×

4962

    void*   buf = taosMemoryCalloc(1, len);

×

4963

    if (!buf) {

×

4964

      qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY));

×

4965

      return;

×

4967

    void* pBuf = buf;

×

4968

    len = doStreamStateEncodeOpState(&pBuf, len, pOperator, true);

×

4969

    pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_CHECKPOINT_NAME,

×

4971

    taosMemoryFree(buf);

×

4972

    saveStreamOperatorStateComplete(&pInfo->basic);

×

4976

static int32_t buildStateResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

4977

  int32_t                      code = TSDB_CODE_SUCCESS;

×

4978

  int32_t                      lino = 0;

×

4979

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

4980

  SOptrBasicInfo*              pBInfo = &pInfo->binfo;

×

4981

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

4982

  SStreamNotifyEventSupp*      pNotifySup = &pInfo->basic.notifyEventSup;

×

4983

  STaskNotifyEventStat*        pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;

×

4984

  bool                         addNotifyEvent = false;

×

4985

  addNotifyEvent = BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);

×

4986

  doBuildDeleteDataBlock(pOperator, pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator, &pInfo->groupResInfo);

×

4987

  if (pInfo->pDelRes->info.rows > 0) {

×

4988

    printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

4989

    if (addNotifyEvent) {

×

4990

      code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);

×

4991

      QUERY_CHECK_CODE(code, lino, _end);

×

4993

    (*ppRes) = pInfo->pDelRes;

×

4994

    return code;

×

4997

  doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes,

×

4999

  if (pBInfo->pRes->info.rows > 0) {

×

5000

    printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5001

    if (addNotifyEvent) {

×

5002

      code = addAggResultNotifyEvent(pBInfo->pRes, pNotifySup->pSessionKeys, pTaskInfo->streamInfo.notifyResultSchema,

×

5004

      QUERY_CHECK_CODE(code, lino, _end);

×

5006

    (*ppRes) = pBInfo->pRes;

×

5007

    return code;

×

5010

  code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);

×

5011

  QUERY_CHECK_CODE(code, lino, _end);

×

5012

  if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {

×

5013

    printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5014

    (*ppRes) = pNotifySup->pEventBlock;

×

5015

    return code;

×

5018

  code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);

×

5019

  QUERY_CHECK_CODE(code, lino, _end);

×

5021

_end:

×

5022

  if (code != TSDB_CODE_SUCCESS) {

×

5023

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

5025

  (*ppRes) = NULL;

×

5026

  return code;

×

5029

static int32_t doStreamStateAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

5030

  if (pOperator->status == OP_EXEC_DONE) {

×

5031

    (*ppRes) = NULL;

×

5032

    return TSDB_CODE_SUCCESS;

×

5035

  int32_t                      code = TSDB_CODE_SUCCESS;

×

5036

  int32_t                      lino = 0;

×

5037

  SExprSupp*                   pSup = &pOperator->exprSupp;

×

5038

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

5039

  SOptrBasicInfo*              pBInfo = &pInfo->binfo;

×

5040

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

5041

  qDebug("===stream=== stream state agg");

×

5042

  if (pOperator->status == OP_RES_TO_RETURN) {

×

5043

    SSDataBlock* resBlock = NULL;

×

5044

    code = buildStateResult(pOperator, &resBlock);

×

5045

    QUERY_CHECK_CODE(code, lino, _end);

×

5046

    if (resBlock != NULL) {

×

5047

      (*ppRes) = resBlock;

×

5048

      return code;

×

5051

    if (pInfo->recvGetAll) {

×

5052

      pInfo->recvGetAll = false;

×

5053

      resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows);

×

5056

    if (pInfo->reCkBlock) {

×

5057

      pInfo->reCkBlock = false;

×

5058

      printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5059

      (*ppRes) = pInfo->pCheckpointRes;

×

5060

      return code;

×

5063

    setStreamOperatorCompleted(pOperator);

×

5064

    (*ppRes) = NULL;

×

5065

    return code;

×

5068

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

5069

  if (!pInfo->pUpdated) {

×

5070

    pInfo->pUpdated = taosArrayInit(16, sizeof(SResultWindowInfo));

×

5071

    QUERY_CHECK_NULL(pInfo->pUpdated, code, lino, _end, terrno);

×

5073

  if (!pInfo->pSeUpdated) {

×

5074

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

5075

    pInfo->pSeUpdated = tSimpleHashInit(64, hashFn);

×

5076

    QUERY_CHECK_NULL(pInfo->pSeUpdated, code, lino, _end, terrno);

×

5078

  while (1) {

×

5079

    SSDataBlock* pBlock = NULL;

×

5080

    code = downstream->fpSet.getNextFn(downstream, &pBlock);

×

5081

    QUERY_CHECK_CODE(code, lino, _end);

×

5083

    if (pBlock == NULL) {

×

5084

      break;

×

5086

    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));

×

5087

    setStreamOperatorState(&pInfo->basic, pBlock->info.type);

×

5089

    if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||

×

5090

        pBlock->info.type == STREAM_CLEAR) {

×

5091

      bool add = pInfo->destHasPrimaryKey && IS_NORMAL_STATE_OP(pOperator);

×

5092

      code = deleteSessionWinState(&pInfo->streamAggSup, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted,

×

5094

      QUERY_CHECK_CODE(code, lino, _end);

×

5095

      continue;

×

5096

    } else if (pBlock->info.type == STREAM_GET_ALL) {

×

5097

      pInfo->recvGetAll = true;

×

5098

      code = getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pSeUpdated);

×

5099

      QUERY_CHECK_CODE(code, lino, _end);

×

5100

      continue;

×

5101

    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {

×

5102

      (*ppRes) = pBlock;

×

5103

      return code;

×

5104

    } else if (pBlock->info.type == STREAM_CHECKPOINT) {

×

5105

      pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState);

×

5106

      doStreamStateSaveCheckpoint(pOperator);

×

5107

      code = copyDataBlock(pInfo->pCheckpointRes, pBlock);

×

5108

      QUERY_CHECK_CODE(code, lino, _end);

×

5110

      continue;

×

5112

      if (pBlock->info.type != STREAM_NORMAL && pBlock->info.type != STREAM_INVALID) {

×

5113

        code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;

×

5114

        QUERY_CHECK_CODE(code, lino, _end);

×

5118

    if (pInfo->scalarSupp.pExprInfo != NULL) {

×

5119

      SExprSupp* pExprSup = &pInfo->scalarSupp;

×

5120

      code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);

×

5121

      QUERY_CHECK_CODE(code, lino, _end);

×

5124

    code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

5125

    QUERY_CHECK_CODE(code, lino, _end);

×

5126

    doStreamStateAggImpl(pOperator, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted);

×

5127

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);

×

5130

  pOperator->status = OP_RES_TO_RETURN;

×

5132

  code = closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pInfo->pSeUpdated);

×

5133

  QUERY_CHECK_CODE(code, lino, _end);

×

5135

  code = copyUpdateResult(&pInfo->pSeUpdated, pInfo->pUpdated, sessionKeyCompareAsc);

×

5136

  QUERY_CHECK_CODE(code, lino, _end);

×

5138

  removeSessionDeleteResults(pInfo->pSeDeleted, pInfo->pUpdated);

×

5140

  if (pInfo->isHistoryOp) {

×

5141

    code = getMaxTsWins(pInfo->pUpdated, pInfo->historyWins);

×

5142

    QUERY_CHECK_CODE(code, lino, _end);

×

5144

  if (pInfo->destHasPrimaryKey && IS_NORMAL_STATE_OP(pOperator)) {

×

5145

    code = copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pSeDeleted);

×

5146

    QUERY_CHECK_CODE(code, lino, _end);

×

5149

  initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);

×

5150

  pInfo->pUpdated = NULL;

×

5151

  code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);

×

5152

  QUERY_CHECK_CODE(code, lino, _end);

×

5154

  SSDataBlock* resBlock = NULL;

×

5155

  code = buildStateResult(pOperator, &resBlock);

×

5156

  QUERY_CHECK_CODE(code, lino, _end);

×

5157

  if (resBlock != NULL) {

×

5158

    (*ppRes) = resBlock;

×

5159

    return code;

×

5162

_end:

×

5163

  if (code != TSDB_CODE_SUCCESS) {

×

5164

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

5165

    pTaskInfo->code = code;

×

5166

    T_LONG_JMP(pTaskInfo->env, code);

×

5168

  setStreamOperatorCompleted(pOperator);

×

5169

  (*ppRes) = NULL;

×

5170

  return code;

×

5173

static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {

×

5174

  SSDataBlock* pRes = NULL;

×

5175

  int32_t      code = doStreamStateAggNext(pOperator, &pRes);

×

5176

  return pRes;

×

5179

void streamStateReleaseState(SOperatorInfo* pOperator) {

×

5180

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

5181

  int32_t                      winSize = taosArrayGetSize(pInfo->historyWins) * sizeof(SSessionKey);

×

5182

  int32_t                      resSize = winSize + sizeof(TSKEY);

×

5183

  char*                        pBuff = taosMemoryCalloc(1, resSize);

×

5184

  if (!pBuff) {

×

5185

    return;

×

5187

  memcpy(pBuff, pInfo->historyWins->pData, winSize);

×

5188

  memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY));

×

5189

  qDebug("===stream=== relase state. save result count:%d", (int32_t)taosArrayGetSize(pInfo->historyWins));

×

5190

  pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_STATE_NAME,

×

5192

  pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState);

×

5193

  taosMemoryFreeClear(pBuff);

×

5195

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

5196

  if (downstream->fpSet.releaseStreamStateFn) {

×

5197

    downstream->fpSet.releaseStreamStateFn(downstream);

×

5201

static int32_t compactStateWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin,

×

5203

  SExprSupp*                   pSup = &pOperator->exprSupp;

×

5204

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

5205

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

5206

  return compactTimeWindow(pSup, &pInfo->streamAggSup, &pInfo->twAggSup, pTaskInfo, pCurWin, pNextWin, pStUpdated,

×

5210

void streamStateReloadState(SOperatorInfo* pOperator) {

×

5211

  int32_t                      code = TSDB_CODE_SUCCESS;

×

5212

  int32_t                      lino = 0;

×

5213

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

5214

  SStreamAggSupporter*         pAggSup = &pInfo->streamAggSup;

×

5215

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

5216

  resetWinRange(&pAggSup->winRange);

×

5218

  SSessionKey seKey = {.win.skey = INT64_MIN, .win.ekey = INT64_MIN, .groupId = 0};

×

5219

  int32_t     size = 0;

×

5220

  void*       pBuf = NULL;

×

5221

  code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_STATE_OP_STATE_NAME,

×

5223

  QUERY_CHECK_CODE(code, lino, _end);

×

5225

  int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);

×

5226

  qDebug("===stream=== reload state. get result count:%d", num);

×

5227

  SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;

×

5229

  TSKEY ts = *(TSKEY*)((char*)pBuf + size - sizeof(TSKEY));

×

5230

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);

×

5231

  pAggSup->stateStore.streamStateReloadInfo(pAggSup->pState, ts);

×

5233

  if (!pInfo->pSeUpdated && num > 0) {

×

5234

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

5235

    pInfo->pSeUpdated = tSimpleHashInit(64, hashFn);

×

5236

    QUERY_CHECK_NULL(pInfo->pSeUpdated, code, lino, _end, terrno);

×

5238

  if (!pInfo->pSeDeleted && num > 0) {

×

5239

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

5240

    pInfo->pSeDeleted = tSimpleHashInit(64, hashFn);

×

5241

    QUERY_CHECK_NULL(pInfo->pSeDeleted, code, lino, _end, terrno);

×

5243

  for (int32_t i = 0; i < num; i++) {

×

5244

    SStateWindowInfo curInfo = {0};

×

5245

    SStateWindowInfo nextInfo = {0};

×

5246

    qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey,

×

5248

    code = getStateWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo, &nextInfo);

×

5249

    QUERY_CHECK_CODE(code, lino, _end);

×

5251

    bool cpRes = compareWinStateKey(curInfo.pStateKey, nextInfo.pStateKey);

×

5252

    qDebug("===stream=== reload state. next window info %" PRId64 ", %" PRIu64 ", compare:%d",

×

5254

    if (cpRes) {

×

5255

      code = compactStateWindow(pOperator, &curInfo.winInfo, &nextInfo.winInfo, pInfo->pSeUpdated, pInfo->pSeDeleted);

×

5256

      qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey,

×

5258

      QUERY_CHECK_CODE(code, lino, _end);

×

5260

      if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {

×

5261

        code = saveResult(curInfo.winInfo, pInfo->pSeUpdated);

×

5262

        QUERY_CHECK_CODE(code, lino, _end);

×

5263

      } else if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

5264

        if (!isCloseWindow(&curInfo.winInfo.sessionWin.win, &pInfo->twAggSup)) {

×

5265

          code = saveDeleteRes(pInfo->pSeDeleted, curInfo.winInfo.sessionWin);

×

5266

          QUERY_CHECK_CODE(code, lino, _end);

×

5268

        SSessionKey key = {0};

×

5269

        getSessionHashKey(&curInfo.winInfo.sessionWin, &key);

×

5270

        code = tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curInfo.winInfo,

×

5272

        QUERY_CHECK_CODE(code, lino, _end);

×

5274

    } else if (IS_VALID_SESSION_WIN(nextInfo.winInfo)) {

×

5275

      releaseOutputBuf(pAggSup->pState, nextInfo.winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);

×

5278

    if (IS_VALID_SESSION_WIN(curInfo.winInfo)) {

×

5279

      code = saveSessionOutputBuf(pAggSup, &curInfo.winInfo);

×

5280

      QUERY_CHECK_CODE(code, lino, _end);

×

5283

  taosMemoryFreeClear(pBuf);

×

5285

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

5286

  if (downstream->fpSet.reloadStreamStateFn) {

×

5287

    downstream->fpSet.reloadStreamStateFn(downstream);

×

5289

  reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup);

×

5291

_end:

×

5292

  taosMemoryFreeClear(pBuf);

×

5293

  if (code != TSDB_CODE_SUCCESS) {

×

5294

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

5296

×

5298

int32_t createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,

×

5300

  QRY_PARAM_CHECK(pOptrInfo);

×

5301

  int32_t code = 0;

×

5302

  int32_t lino = 0;

×

5304

  SStreamStateWinodwPhysiNode* pStateNode = (SStreamStateWinodwPhysiNode*)pPhyNode;

×

5305

  int32_t                      tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;

×

5306

  SColumnNode*                 pColNode = (SColumnNode*)(pStateNode->pStateKey);

×

5307

  SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo));

×

5308

  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));

×

5309

  if (pInfo == NULL || pOperator == NULL) {

×

5310

    code = terrno;

×

5311

    QUERY_CHECK_CODE(code, lino, _error);

×

5314

  pInfo->stateCol = extractColumnFromColumnNode(pColNode);

×

5315

  initResultSizeInfo(&pOperator->resultInfo, 4096);

×

5316

  if (pStateNode->window.pExprs != NULL) {

×

5317

    int32_t    numOfScalar = 0;

×

5318

    SExprInfo* pScalarExprInfo = NULL;

×

5319

    code = createExprInfo(pStateNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalar);

×

5320

    QUERY_CHECK_CODE(code, lino, _error);

×

5322

    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);

×

5323

    QUERY_CHECK_CODE(code, lino, _error);

×

5326

  pInfo->twAggSup = (STimeWindowAggSupp){

×

5327

      .waterMark = pStateNode->window.watermark,

×

5328

      .calTrigger = pStateNode->window.triggerType,

×

5331

      .deleteMark = getDeleteMark(&pStateNode->window, 0),

×

5334

  code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

×

5335

  QUERY_CHECK_CODE(code, lino, _error);

×

5337

  SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);

×

5338

  QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);

×

5339

  pInfo->binfo.pRes = pResBlock;

×

5341

  SExprSupp* pExpSup = &pOperator->exprSupp;

×

5342

  int32_t    numOfCols = 0;

×

5343

  SExprInfo* pExprInfo = NULL;

×

5344

  code = createExprInfo(pStateNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);

×

5345

  QUERY_CHECK_CODE(code, lino, _error);

×

5347

  code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore);

×

5348

  if (code != TSDB_CODE_SUCCESS) {

×

5349

    goto _error;

×

5351

  int32_t keySize = sizeof(SStateKeys) + pColNode->node.resType.bytes;

×

5352

  int16_t type = pColNode->node.resType.type;

×

5353

  pInfo->primaryTsIndex = tsSlotId;

×

5355

      initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState, keySize, type,

×

5356

                             &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup, GET_TASKID(pTaskInfo),

×

5358

  QUERY_CHECK_CODE(code, lino, _error);

×

5360

  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

5361

  pInfo->pSeDeleted = tSimpleHashInit(64, hashFn);

×

5362

  QUERY_CHECK_NULL(pInfo->pSeDeleted, code, lino, _error, terrno);

×

5363

  pInfo->pDelIterator = NULL;

×

5365

  code = createSpecialDataBlock(STREAM_DELETE_RESULT, &pInfo->pDelRes);

×

5366

  QUERY_CHECK_CODE(code, lino, _error);

×

5368

  pInfo->pChildren = NULL;

×

5369

  pInfo->ignoreExpiredData = pStateNode->window.igExpired;

×

5370

  pInfo->ignoreExpiredDataSaved = false;

×

5371

  pInfo->pUpdated = NULL;

×

5372

  pInfo->pSeUpdated = NULL;

×

5373

  pInfo->dataVersion = 0;

×

5374

  pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey));

×

5375

  if (!pInfo->historyWins) {

×

5376

    code = terrno;

×

5377

    QUERY_CHECK_CODE(code, lino, _error);

×

5380

  if (pHandle) {

×

5381

    pInfo->isHistoryOp = pHandle->fillHistory;

×

5384

  code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);

×

5385

  QUERY_CHECK_CODE(code, lino, _error);

×

5387

  pInfo->recvGetAll = false;

×

5388

  pInfo->pPkDeleted = tSimpleHashInit(64, hashFn);

×

5389

  QUERY_CHECK_NULL(pInfo->pPkDeleted, code, lino, _error, terrno);

×

5390

  pInfo->destHasPrimaryKey = pStateNode->window.destHasPrimaryKey;

×

5391

  pInfo->pOperator = pOperator;

×

5393

  setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED,

×

5396

  void*   buff = NULL;

×

5397

  int32_t len = 0;

×

5399

      pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_CHECKPOINT_NAME,

×

5401

  if (res == TSDB_CODE_SUCCESS) {

×

5402

    code = doStreamStateDecodeOpState(buff, len, pOperator, true, NULL);

×

5403

    taosMemoryFree(buff);

×

5404

    QUERY_CHECK_CODE(code, lino, _error);

×

5407

  pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAggNext, NULL, destroyStreamStateOperatorInfo,

×

5409

  setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState);

×

5411

  code = initStreamBasicInfo(&pInfo->basic, pOperator);

×

5412

  QUERY_CHECK_CODE(code, lino, _error);

×

5414

  code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,

×

5415

                        &pInfo->twAggSup, &pInfo->basic);

×

5416

  QUERY_CHECK_CODE(code, lino, _error);

×

5418

  code = appendDownstream(pOperator, &downstream, 1);

×

5419

  QUERY_CHECK_CODE(code, lino, _error);

×

5421

  pInfo->trueForLimit = pStateNode->trueForLimit;

×

5423

  *pOptrInfo = pOperator;

×

5424

  return TSDB_CODE_SUCCESS;

×

5426

_error:

×

5427

  if (pInfo != NULL) destroyStreamStateOperatorInfo(pInfo);

×

5428

  destroyOperatorAndDownstreams(pOperator, &downstream, 1);

×

5429

  pTaskInfo->code = code;

×

5430

  qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

5431

  return code;

×

5446

static int32_t doStreamIntervalAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

5447

  int32_t                      code = TSDB_CODE_SUCCESS;

×

5448

  int32_t                      lino = 0;

×

5449

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

5450

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

5451

  SStorageAPI*                 pAPI = &pOperator->pTaskInfo->storageAPI;

×

5452

  SExprSupp*                   pSup = &pOperator->exprSupp;

×

5454

  qDebug("stask:%s  %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);

×

5456

  if (pOperator->status == OP_EXEC_DONE) {

×

5457

    (*ppRes) = NULL;

×

5458

    return code;

×

5461

  if (pOperator->status == OP_RES_TO_RETURN) {

×

5462

    SSDataBlock* resBlock = NULL;

×

5463

    code = buildIntervalResult(pOperator, &resBlock);

×

5464

    QUERY_CHECK_CODE(code, lino, _end);

×

5465

    if (resBlock != NULL) {

×

5466

      (*ppRes) = resBlock;

×

5467

      return code;

×

5470

    if (pInfo->recvGetAll) {

×

5471

      pInfo->recvGetAll = false;

×

5472

      resetUnCloseWinInfo(pInfo->aggSup.pResultRowHashTable);

×

5475

    if (pInfo->reCkBlock) {

×

5476

      pInfo->reCkBlock = false;

×

5477

      printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5478

      (*ppRes) = pInfo->pCheckpointRes;

×

5479

      return code;

×

5482

    setStreamOperatorCompleted(pOperator);

×

5483

    (*ppRes) = NULL;

×

5484

    return code;

×

5487

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

5489

  if (!pInfo->pUpdated) {

×

5490

    pInfo->pUpdated = taosArrayInit(4096, POINTER_BYTES);

×

5491

    QUERY_CHECK_NULL(pInfo->pUpdated, code, lino, _end, terrno);

×

5494

  if (!pInfo->pUpdatedMap) {

×

5495

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

5496

    pInfo->pUpdatedMap = tSimpleHashInit(4096, hashFn);

×

5497

    QUERY_CHECK_NULL(pInfo->pUpdatedMap, code, lino, _end, terrno);

×

5500

  while (1) {

×

5501

    SSDataBlock* pBlock = NULL;

×

5502

    code = downstream->fpSet.getNextFn(downstream, &pBlock);

×

5503

    QUERY_CHECK_CODE(code, lino, _end);

×

5505

    if (pBlock == NULL) {

×

5506

      qDebug("===stream===return data:%s. recv datablock num:%" PRIu64, getStreamOpName(pOperator->operatorType),

×

5508

      pInfo->numOfDatapack = 0;

×

5509

      break;

×

5512

    pInfo->numOfDatapack++;

×

5513

    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));

×

5514

    setStreamOperatorState(&pInfo->basic, pBlock->info.type);

×

5516

    if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||

×

5517

        pBlock->info.type == STREAM_CLEAR) {

×

5518

      code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, pInfo->pDelWins, pInfo->pUpdatedMap, NULL);

×

5519

      QUERY_CHECK_CODE(code, lino, _end);

×

5520

      continue;

×

5521

    } else if (pBlock->info.type == STREAM_GET_ALL) {

×

5522

      pInfo->recvGetAll = true;

×

5523

      code = getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap);

×

5524

      QUERY_CHECK_CODE(code, lino, _end);

×

5525

      continue;

×

5526

    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {

×

5527

      printDataBlock(pBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5528

      (*ppRes) = pBlock;

×

5529

      return code;

×

5530

    } else if (pBlock->info.type == STREAM_DROP_CHILD_TABLE) {

×

5531

      doDeleteWindowByGroupId(pOperator, pBlock);

×

5532

      printDataBlock(pBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5533

      (*ppRes) = pBlock;

×

5534

      return code;

×

5535

    } else if (pBlock->info.type == STREAM_CHECKPOINT) {

×

5536

      pAPI->stateStore.streamStateCommit(pInfo->pState);

×

5537

      doStreamIntervalSaveCheckpoint(pOperator);

×

5538

      pInfo->reCkBlock = true;

×

5539

      code = copyDataBlock(pInfo->pCheckpointRes, pBlock);

×

5540

      QUERY_CHECK_CODE(code, lino, _end);

×

5542

      continue;

×

5544

      if (pBlock->info.type != STREAM_NORMAL && pBlock->info.type != STREAM_INVALID) {

×

5545

        code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;

×

5546

        QUERY_CHECK_CODE(code, lino, _end);

×

5550

    if (pBlock->info.type == STREAM_NORMAL && pBlock->info.version != 0) {

×

5552

      pTaskInfo->version = pBlock->info.version;

×

5555

    if (pInfo->scalarSupp.pExprInfo != NULL) {

×

5556

      SExprSupp* pExprSup = &pInfo->scalarSupp;

×

5557

      code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);

×

5558

      QUERY_CHECK_CODE(code, lino, _end);

×

5564

    code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

5565

    QUERY_CHECK_CODE(code, lino, _end);

×

5572

    code = doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pInfo->pUpdatedMap, pInfo->pDeletedMap);

×

5573

    if (code == TSDB_CODE_STREAM_INTERNAL_ERROR) {

×

5574

      pOperator->status = OP_RES_TO_RETURN;

×

5575

      code = TSDB_CODE_SUCCESS;

×

5576

      break;

×

5578

    QUERY_CHECK_CODE(code, lino, _end);

×

5579

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);

×

5580

    pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey);

×

5582

  pOperator->status = OP_RES_TO_RETURN;

×

5583

  if (!pInfo->destHasPrimaryKey) {

×

5584

    removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins);

×

5586

  code = closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL,

×

5588

  QUERY_CHECK_CODE(code, lino, _end);

×

5590

  if (pInfo->destHasPrimaryKey && IS_NORMAL_INTERVAL_OP(pOperator)) {

×

5591

    code = copyIntervalDeleteKey(pInfo->pDeletedMap, pInfo->pDelWins);

×

5592

    QUERY_CHECK_CODE(code, lino, _end);

×

5595

  void*   pIte = NULL;

×

5596

  int32_t iter = 0;

×

5597

  while ((pIte = tSimpleHashIterate(pInfo->pUpdatedMap, pIte, &iter)) != NULL) {

×

5598

    void* tmp = taosArrayPush(pInfo->pUpdated, pIte);

×

5599

    if (!tmp) {

×

5600

      code = terrno;

×

5601

      QUERY_CHECK_CODE(code, lino, _end);

×

5604

  taosArraySort(pInfo->pUpdated, winPosCmprImpl);

×

5606

  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);

×

5607

  pInfo->pUpdated = NULL;

×

5608

  code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);

×

5609

  QUERY_CHECK_CODE(code, lino, _end);

×

5611

  tSimpleHashCleanup(pInfo->pUpdatedMap);

×

5612

  pInfo->pUpdatedMap = NULL;

×

5614

  code = buildIntervalResult(pOperator, ppRes);

×

5615

  QUERY_CHECK_CODE(code, lino, _end);

×

5617

  return code;

×

5619

_end:

×

5620

  if (code != TSDB_CODE_SUCCESS) {

×

5621

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

5622

    pTaskInfo->code = code;

×

5623

    T_LONG_JMP(pTaskInfo->env, code);

×

5625

  setStreamOperatorCompleted(pOperator);

×

5626

  (*ppRes) = NULL;

×

5627

  return code;

×

5630

static int32_t createStreamSingleIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,

×

5633

  QRY_PARAM_CHECK(pOptrInfo);

×

5635

  int32_t code = TSDB_CODE_SUCCESS;

×

5636

  int32_t lino = 0;

×

5637

  int32_t numOfCols = 0;

×

5639

  SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo));

×

5640

  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));

×

5641

  if (pInfo == NULL || pOperator == NULL) {

×

5642

    code = terrno;

×

5643

    QUERY_CHECK_CODE(code, lino, _error);

×

5646

  SStreamIntervalPhysiNode* pIntervalPhyNode = (SStreamIntervalPhysiNode*)pPhyNode;

×

5648

  SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);

×

5649

  QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);

×

5650

  initBasicInfo(&pInfo->binfo, pResBlock);

×

5652

  pInfo->interval = (SInterval){

×

5653

      .interval = pIntervalPhyNode->interval,

×

5654

      .sliding = pIntervalPhyNode->sliding,

×

5655

      .intervalUnit = pIntervalPhyNode->intervalUnit,

×

5656

      .slidingUnit = pIntervalPhyNode->slidingUnit,

×

5657

      .offset = pIntervalPhyNode->offset,

×

5658

      .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision,

×

5659

      .timeRange = pIntervalPhyNode->timeRange,

×

5661

  calcIntervalAutoOffset(&pInfo->interval);

×

5663

  pInfo->twAggSup =

×

5664

      (STimeWindowAggSupp){.waterMark = pIntervalPhyNode->window.watermark,

×

5665

                           .calTrigger = pIntervalPhyNode->window.triggerType,

×

5668

                           .deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval)};

×

5670

  pOperator->pTaskInfo = pTaskInfo;

×

5671

  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;

×

5673

  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;

×

5674

  pInfo->ignoreExpiredDataSaved = false;

×

5676

  SExprSupp* pSup = &pOperator->exprSupp;

×

5677

  pSup->hasWindowOrGroup = true;

×

5679

  code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

×

5680

  QUERY_CHECK_CODE(code, lino, _error);

×

5682

  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;

×

5683

  initResultSizeInfo(&pOperator->resultInfo, 4096);

×

5685

  pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState));

×

5686

  QUERY_CHECK_NULL(pInfo->pState, code, lino, _error, terrno);

×

5687

  *(pInfo->pState) = *(pTaskInfo->streamInfo.pState);

×

5688

  pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsIndex);

×

5690

  size_t     keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;

×

5691

  SExprInfo* pExprInfo = NULL;

×

5692

  code = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);

×

5693

  QUERY_CHECK_CODE(code, lino, _error);

×

5694

  code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, pInfo->pState,

×

5696

  QUERY_CHECK_CODE(code, lino, _error);

×

5698

  if (pIntervalPhyNode->window.pExprs != NULL) {

×

5699

    int32_t    numOfScalar = 0;

×

5700

    SExprInfo* pScalarExprInfo = NULL;

×

5702

    code = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalar);

×

5703

    QUERY_CHECK_CODE(code, lino, _error);

×

5705

    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);

×

5706

    QUERY_CHECK_CODE(code, lino, _error);

×

5709

  pInfo->invertible = false;

×

5710

  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));

×

5711

  QUERY_CHECK_NULL(pInfo->pDelWins, code, lino, _error, terrno);

×

5712

  pInfo->delIndex = 0;

×

5714

  code = createSpecialDataBlock(STREAM_DELETE_RESULT, &pInfo->pDelRes);

×

5715

  QUERY_CHECK_CODE(code, lino, _error);

×

5717

  initResultRowInfo(&pInfo->binfo.resultRowInfo);

×

5719

  pInfo->pPhyNode = NULL;  // create new child

×

5720

  pInfo->pPullDataMap = NULL;

×

5721

  pInfo->pFinalPullDataMap = NULL;

×

5722

  pInfo->pPullWins = NULL;  // SPullWindowInfo

×

5723

  pInfo->pullIndex = 0;

×

5724

  pInfo->pPullDataRes = NULL;

×

5725

  pInfo->numOfChild = 0;

×

5726

  pInfo->delKey.ts = INT64_MAX;

×

5727

  pInfo->delKey.groupId = 0;

×

5728

  pInfo->numOfDatapack = 0;

×

5729

  pInfo->pUpdated = NULL;

×

5730

  pInfo->pUpdatedMap = NULL;

×

5731

  int32_t funResSize = getMaxFunResSize(pSup, numOfCols);

×

5733

  pInfo->stateStore = pTaskInfo->storageAPI.stateStore;

×

5734

  pInfo->pState->pFileState = NULL;

×

5737

  pInfo->pState->pResultRowStore.resultRowGet = getResultRowFromBuf;

×

5738

  pInfo->pState->pResultRowStore.resultRowPut = putResultRowToBuf;

×

5739

  pInfo->pState->pExprSupp = &pOperator->exprSupp;

×

5741

  code = pTaskInfo->storageAPI.stateStore.streamFileStateInit(

×

5742

      tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, compareTs, pInfo->pState,

×

5743

      pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo), pHandle->checkpointId, STREAM_STATE_BUFF_HASH,

×

5744

      &pInfo->pState->pFileState);

×

5745

  QUERY_CHECK_CODE(code, lino, _error);

×

5747

  pInfo->pOperator = pOperator;

×

5748

  setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED,

×

5751

      createOperatorFpSet(optrDummyOpenFn, doStreamIntervalAggNext, NULL, destroyStreamFinalIntervalOperatorInfo,

×

5753

  setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState);

×

5755

  code = initStreamBasicInfo(&pInfo->basic, pOperator);

×

5756

  QUERY_CHECK_CODE(code, lino, _error);

×

5758

  pInfo->recvGetAll = false;

×

5760

  code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);

×

5761

  QUERY_CHECK_CODE(code, lino, _error);

×

5763

  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

5764

  pInfo->pDeletedMap = tSimpleHashInit(4096, hashFn);

×

5765

  QUERY_CHECK_NULL(pInfo->pDeletedMap, code, lino, _error, terrno);

×

5766

  pInfo->destHasPrimaryKey = pIntervalPhyNode->window.destHasPrimaryKey;

×

5769

  void*   buff = NULL;

×

5770

  int32_t len = 0;

×

5771

  int32_t res = pAPI->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_CHECKPOINT_NAME,

×

5773

  if (res == TSDB_CODE_SUCCESS) {

×

5774

    doStreamIntervalDecodeOpState(buff, len, pOperator);

×

5775

    taosMemoryFree(buff);

×

5778

  pInfo->basic.primaryPkIndex = -1;

×

5779

  code = initIntervalDownStream(downstream, pPhyNode->type, pInfo, &pInfo->basic);

×

5780

  QUERY_CHECK_CODE(code, lino, _error);

×

5782

  code = appendDownstream(pOperator, &downstream, 1);

×

5783

  QUERY_CHECK_CODE(code, lino, _error);

×

5785

  *pOptrInfo = pOperator;

×

5786

  return TSDB_CODE_SUCCESS;

×

5788

_error:

×

5789

  if (pInfo != NULL) destroyStreamFinalIntervalOperatorInfo(pInfo);

×

5790

  destroyOperatorAndDownstreams(pOperator, &downstream, 1);

×

5791

  pTaskInfo->code = code;

×

5792

  return code;

×

5795

int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,

×

5797

  SStreamIntervalPhysiNode* pIntervalPhyNode = (SStreamIntervalPhysiNode*)pPhyNode;

×

5798

  if (pIntervalPhyNode->window.triggerType == STREAM_TRIGGER_FORCE_WINDOW_CLOSE) {

×

5799

    return createStreamIntervalSliceOperatorInfo(downstream, pPhyNode, pTaskInfo, pHandle, pOptrInfo);

×

5801

    return createStreamSingleIntervalOperatorInfo(downstream, pPhyNode, pTaskInfo, pHandle, pOptrInfo);

×

5806

static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pUpdatedMap) {

×

5807

  int32_t                      code = TSDB_CODE_SUCCESS;

×

5808

  int32_t                      lino = 0;

×

5809

  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info;

×

5810

  pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);

×

5812

  SResultRowInfo*  pResultRowInfo = &(pInfo->binfo.resultRowInfo);

×

5813

  SExecTaskInfo*   pTaskInfo = pOperator->pTaskInfo;

×

5814

  SExprSupp*       pSup = &pOperator->exprSupp;

×

5815

  int32_t          numOfOutput = pSup->numOfExprs;

×

5816

  int32_t          step = 1;

×

5817

  SRowBuffPos*     pResPos = NULL;

×

5818

  SResultRow*      pResult = NULL;

×

5819

  int32_t          forwardRows = 1;

×

5820

  uint64_t         groupId = pSDataBlock->info.id.groupId;

×

5821

  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);

×

5822

  TSKEY*           tsCol = (int64_t*)pColDataInfo->pData;

×

5824

  int32_t     startPos = 0;

×

5825

  TSKEY       ts = getStartTsKey(&pSDataBlock->info.window, tsCol);

×

5826

  STimeWindow nextWin = getFinalTimeWindow(ts, &pInfo->interval);

×

5828

  while (1) {

×

5829

    SWinKey key = {

×

5830

        .ts = nextWin.skey,

×

5833

    void*   chIds = taosHashGet(pInfo->pPullDataMap, &key, sizeof(SWinKey));

×

5834

    int32_t index = -1;

×

5835

    SArray* chArray = NULL;

×

5836

    int32_t chId = 0;

×

5837

    if (chIds) {

×

5838

      chArray = *(void**)chIds;

×

5839

      chId = getChildIndex(pSDataBlock);

×

5840

      index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);

×

5842

    if (!(index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA)) {

×

5843

      startPos = getNextQualifiedFinalWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCol, startPos);

×

5844

      if (startPos < 0) {

×

5845

        break;

×

5847

      continue;

×

5850

    if (!inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) {

×

5851

      startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCol, startPos, nextWin.ekey, &nextWin);

×

5852

      if (startPos < 0) {

×

5853

        break;

×

5855

      continue;

×

5858

    int32_t winCode = TSDB_CODE_SUCCESS;

×

5859

    code = setIntervalOutputBuf(pInfo->pState, &nextWin, &pResPos, groupId, pSup->pCtx, numOfOutput,

×

5861

    QUERY_CHECK_CODE(code, lino, _end);

×

5863

    pResult = (SResultRow*)pResPos->pRowBuff;

×

5865

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {

×

5866

      code = saveWinResult(&key, pResPos, pUpdatedMap);

×

5867

      QUERY_CHECK_CODE(code, lino, _end);

×

5870

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

5871

      code = tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pResPos, POINTER_BYTES);

×

5872

      QUERY_CHECK_CODE(code, lino, _end);

×

5875

    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, 1);

×

5876

    code = applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos,

×

5877

                                           forwardRows, pSDataBlock->info.rows, numOfOutput);

×

5878

    QUERY_CHECK_CODE(code, lino, _end);

×

5879

    key.ts = nextWin.skey;

×

5881

    if (pInfo->delKey.ts > key.ts) {

×

5882

      pInfo->delKey = key;

×

5884

    int32_t prevEndPos = (forwardRows - 1) * step + startPos;

×

5885

    if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) {

×

5886

      qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64

×

5889

      code = blockDataUpdateTsWindow(pSDataBlock, 0);

×

5890

      QUERY_CHECK_CODE(code, lino, _end);

×

5893

      if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) {

×

5894

        qError("table uid %" PRIu64 " data block timestamp is out of range! minKey %" PRId64 ",maxKey %" PRId64,

×

5898

    startPos = getNextQualifiedFinalWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCol, prevEndPos);

×

5899

    if (startPos < 0) {

×

5900

      break;

×

5904

_end:

×

5905

  if (code != TSDB_CODE_SUCCESS) {

×

5906

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

5908

×

5910

static int32_t addMidRetriveWindow(SArray* wins, SHashObj* pMidPullMap, int32_t numOfChild) {

×

5911

  int32_t code = TSDB_CODE_SUCCESS;

×

5912

  int32_t lino = 0;

×

5913

  int32_t size = taosArrayGetSize(wins);

×

5914

  for (int32_t i = 0; i < size; i++) {

×

5915

    SWinKey* winKey = taosArrayGet(wins, i);

×

5916

    void*    chIds = taosHashGet(pMidPullMap, winKey, sizeof(SWinKey));

×

5917

    if (!chIds) {

×

5918

      code = addPullWindow(pMidPullMap, winKey, numOfChild);

×

5919

      qDebug("===stream===prepare mid operator retrive for delete %" PRId64 ", size:%d", winKey->ts, numOfChild);

×

5920

      QUERY_CHECK_CODE(code, lino, _end);

×

5923

_end:

×

5924

  if (code != TSDB_CODE_SUCCESS) {

×

5925

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

5927

  return code;

×

5930

static SSDataBlock* buildMidIntervalResult(SOperatorInfo* pOperator) {

×

5931

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

5932

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

5933

  uint16_t                     opType = pOperator->operatorType;

×

5935

  if (pInfo->recvPullover) {

×

5936

    pInfo->recvPullover = false;

×

5937

    printDataBlock(pInfo->pMidPulloverRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5938

    return pInfo->pMidPulloverRes;

×

5941

  qDebug("===stream=== build mid interval result");

×

5942

  doBuildDeleteResult(pInfo, pInfo->pMidPullDatas, &pInfo->midDelIndex, pInfo->pDelRes);

×

5943

  if (pInfo->pDelRes->info.rows != 0) {

×

5945

    printDataBlock(pInfo->pDelRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));

×

5946

    return pInfo->pDelRes;

×

5949

  if (pInfo->recvRetrive) {

×

5950

    pInfo->recvRetrive = false;

×

5951

    printDataBlock(pInfo->pMidRetriveRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5952

    return pInfo->pMidRetriveRes;

×

5955

  return NULL;

×

5958

static int32_t doStreamMidIntervalAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

5959

  int32_t                      code = TSDB_CODE_SUCCESS;

×

5960

  int32_t                      lino = 0;

×

5961

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

5962

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

5963

  SStorageAPI*                 pAPI = &pOperator->pTaskInfo->storageAPI;

×

5964

  SOperatorInfo*               downstream = pOperator->pDownstream[0];

×

5965

  SExprSupp*                   pSup = &pOperator->exprSupp;

×

5967

  qDebug("stask:%s  %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);

×

5969

  if (pOperator->status == OP_EXEC_DONE) {

×

5970

    (*ppRes) = NULL;

×

5971

    return code;

×

5972

  } else if (pOperator->status == OP_RES_TO_RETURN) {

×

5973

    SSDataBlock* resBlock = NULL;

×

5974

    code = buildIntervalResult(pOperator, &resBlock);

×

5975

    QUERY_CHECK_CODE(code, lino, _end);

×

5976

    if (resBlock != NULL) {

×

5977

      (*ppRes) = resBlock;

×

5978

      return code;

×

5981

    setOperatorCompleted(pOperator);

×

5982

    clearFunctionContext(&pOperator->exprSupp);

×

5983

    clearStreamIntervalOperator(pInfo);

×

5984

    qDebug("stask:%s  ===stream===%s clear", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType));

×

5985

    (*ppRes) = NULL;

×

5986

    return code;

×

5988

    SSDataBlock* resBlock = NULL;

×

5989

    code = buildIntervalResult(pOperator, &resBlock);

×

5990

    QUERY_CHECK_CODE(code, lino, _end);

×

5991

    if (resBlock != NULL) {

×

5992

      (*ppRes) = resBlock;

×

5993

      return code;

×

5996

    resBlock = buildMidIntervalResult(pOperator);

×

5997

    if (resBlock != NULL) {

×

5998

      (*ppRes) = resBlock;

×

5999

      return code;

×

6002

    if (pInfo->clearState) {

×

6003

      pInfo->clearState = false;

×

6004

      clearFunctionContext(&pOperator->exprSupp);

×

6005

      clearStreamIntervalOperator(pInfo);

×

6009

  if (!pInfo->pUpdated) {

×

6010

    pInfo->pUpdated = taosArrayInit(4096, POINTER_BYTES);

×

6011

    QUERY_CHECK_NULL(pInfo->pUpdated, code, lino, _end, terrno);

×

6013

  if (!pInfo->pUpdatedMap) {

×

6014

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

6015

    pInfo->pUpdatedMap = tSimpleHashInit(4096, hashFn);

×

6016

    QUERY_CHECK_NULL(pInfo->pUpdatedMap, code, lino, _end, terrno);

×

6019

  while (1) {

×

6020

    if (isTaskKilled(pTaskInfo)) {

×

6021

      qInfo("===stream=== %s task is killed, code %s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code));

×

6022

      (*ppRes) = NULL;

×

6023

      return code;

×

6026

    SSDataBlock* pBlock = NULL;

×

6027

    code = downstream->fpSet.getNextFn(downstream, &pBlock);

×

6028

    QUERY_CHECK_CODE(code, lino, _end);

×

6030

    if (pBlock == NULL) {

×

6031

      pOperator->status = OP_RES_TO_RETURN;

×

6032

      qDebug("===stream===return data:%s. recv datablock num:%" PRIu64, getStreamOpName(pOperator->operatorType),

×

6034

      pInfo->numOfDatapack = 0;

×

6035

      break;

×

6037

    pInfo->numOfDatapack++;

×

6038

    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));

×

6039

    setStreamOperatorState(&pInfo->basic, pBlock->info.type);

×

6041

    if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) {

×

6042

      pInfo->binfo.pRes->info.type = pBlock->info.type;

×

6043

    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||

×

6044

               pBlock->info.type == STREAM_CLEAR) {

×

6045

      SArray* delWins = taosArrayInit(8, sizeof(SWinKey));

×

6046

      if (!delWins) {

×

6047

        code = terrno;

×

6048

        QUERY_CHECK_CODE(code, lino, _end);

×

6051

          doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, pInfo->pFinalPullDataMap);

×

6052

      QUERY_CHECK_CODE(code, lino, _end);

×

6054

      removeResults(delWins, pInfo->pUpdatedMap);

×

6055

      void* tmp = taosArrayAddAll(pInfo->pDelWins, delWins);

×

6056

      if (!tmp && taosArrayGetSize(delWins) > 0) {

×

6057

        code = TSDB_CODE_OUT_OF_MEMORY;

×

6058

        QUERY_CHECK_CODE(code, lino, _end);

×

6060

      taosArrayDestroy(delWins);

×

6062

      doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);

×

6063

      if (pInfo->pDelRes->info.rows != 0) {

×

6065

        printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

6066

        if (pBlock->info.type == STREAM_CLEAR) {

×

6067

          pInfo->pDelRes->info.type = STREAM_CLEAR;

×

6069

          pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;

×

6071

        (*ppRes) = pInfo->pDelRes;

×

6072

        return code;

×

6074

      continue;

×

6075

    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {

×

6076

      (*ppRes) = pBlock;

×

6077

      return code;

×

6078

    } else if (pBlock->info.type == STREAM_PULL_OVER) {

×

6079

      code = processPullOver(pBlock, pInfo->pPullDataMap, pInfo->pFinalPullDataMap, &pInfo->interval, pInfo->pPullWins,

×

6081

      QUERY_CHECK_CODE(code, lino, _end);

×

6083

      if (pInfo->recvPullover) {

×

6084

        code = copyDataBlock(pInfo->pMidPulloverRes, pBlock);

×

6085

        QUERY_CHECK_CODE(code, lino, _end);

×

6087

        pInfo->clearState = true;

×

6088

        break;

×

6090

      continue;

×

6091

    } else if (pBlock->info.type == STREAM_CHECKPOINT) {

×

6092

      pAPI->stateStore.streamStateCommit(pInfo->pState);

×

6093

      doStreamIntervalSaveCheckpoint(pOperator);

×

6094

      code = copyDataBlock(pInfo->pCheckpointRes, pBlock);

×

6095

      QUERY_CHECK_CODE(code, lino, _end);

×

6097

      continue;

×

6098

    } else if (pBlock->info.type == STREAM_MID_RETRIEVE) {

×

6099

      SArray* delWins = taosArrayInit(8, sizeof(SWinKey));

×

6100

      if (!delWins) {

×

6101

        code = terrno;

×

6102

        QUERY_CHECK_CODE(code, lino, _end);

×

6104

      code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, NULL);

×

6105

      QUERY_CHECK_CODE(code, lino, _end);

×

6107

      code = addMidRetriveWindow(delWins, pInfo->pPullDataMap, pInfo->numOfChild);

×

6108

      QUERY_CHECK_CODE(code, lino, _end);

×

6110

      taosArrayDestroy(delWins);

×

6111

      pInfo->recvRetrive = true;

×

6112

      code = copyDataBlock(pInfo->pMidRetriveRes, pBlock);

×

6113

      QUERY_CHECK_CODE(code, lino, _end);

×

6115

      pInfo->pMidRetriveRes->info.type = STREAM_MID_RETRIEVE;

×

6116

      pInfo->clearState = true;

×

6117

      break;

×

6119

      if (pBlock->info.type != STREAM_INVALID) {

×

6120

        code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;

×

6121

        QUERY_CHECK_CODE(code, lino, _end);

×

6125

    if (pInfo->scalarSupp.pExprInfo != NULL) {

×

6126

      SExprSupp* pExprSup = &pInfo->scalarSupp;

×

6127

      code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);

×

6128

      QUERY_CHECK_CODE(code, lino, _end);

×

6130

    code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

6131

    QUERY_CHECK_CODE(code, lino, _end);

×

6132

    doStreamMidIntervalAggImpl(pOperator, pBlock, pInfo->pUpdatedMap);

×

6133

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);

×

6134

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.watermark);

×

6135

    pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey);

×

6138

  removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins);

×

6139

  pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs;

×

6141

  void*   pIte = NULL;

×

6142

  int32_t iter = 0;

×

6143

  while ((pIte = tSimpleHashIterate(pInfo->pUpdatedMap, pIte, &iter)) != NULL) {

×

6144

    void* tmp = taosArrayPush(pInfo->pUpdated, pIte);

×

6145

    if (!tmp) {

×

6146

      code = terrno;

×

6147

      QUERY_CHECK_CODE(code, lino, _end);

×

6151

  tSimpleHashCleanup(pInfo->pUpdatedMap);

×

6152

  pInfo->pUpdatedMap = NULL;

×

6153

  taosArraySort(pInfo->pUpdated, winPosCmprImpl);

×

6155

  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);

×

6156

  pInfo->pUpdated = NULL;

×

6157

  code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);

×

6158

  QUERY_CHECK_CODE(code, lino, _end);

×

6160

  SSDataBlock* resBlock = NULL;

×

6161

  code = buildIntervalResult(pOperator, &resBlock);

×

6162

  QUERY_CHECK_CODE(code, lino, _end);

×

6163

  if (resBlock != NULL) {

×

6164

    (*ppRes) = resBlock;

×

6165

    return code;

×

6168

  resBlock = buildMidIntervalResult(pOperator);

×

6169

  if (resBlock != NULL) {

×

6170

    (*ppRes) = resBlock;

×

6171

    return code;

×

6174

  if (pInfo->clearState) {

×

6175

    pInfo->clearState = false;

×

6176

    clearFunctionContext(&pOperator->exprSupp);

×

6177

    clearStreamIntervalOperator(pInfo);

×

6180

_end:

×

6181

  if (code != TSDB_CODE_SUCCESS) {

×

6182

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

6183

    pTaskInfo->code = code;

×

6184

    T_LONG_JMP(pTaskInfo->env, code);

×

6186

  (*ppRes) = NULL;

×

6187

  return code;

×

6190

static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) {

×

6191

  SSDataBlock* pRes = NULL;

×

6192

  int32_t      code = doStreamMidIntervalAggNext(pOperator, &pRes);

×

6193

  return pRes;

×

6196

void setStreamOperatorCompleted(SOperatorInfo* pOperator) {

×

6197

  qDebug("stask:%s  %s status: %d. set completed", GET_TASKID(pOperator->pTaskInfo),

×

6199

  setOperatorCompleted(pOperator);

×

6200

×

taosdata / TDengine / #3646

Source File
Press 'n' to go to next uncovered line, 'b' for previous

taosdata / TDengine / #3646

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous