taosdata/TDengine | Build 3638 | source/libs/executor/src/streamtimewindowoperator.c | Coveralls

61

static int32_t binarySearchCom(void* keyList, int num, void* pKey, int order, __compare_fn_t comparefn) {

×

62

  int firstPos = 0, lastPos = num - 1, midPos = -1;

×

63

  int numOfRows = 0;

×

65

  if (num <= 0) return -1;

×

66

  if (order == TSDB_ORDER_DESC) {

×

69

      if (comparefn(pKey, keyList, lastPos) >= 0) return lastPos;

×

70

      if (comparefn(pKey, keyList, firstPos) == 0) return firstPos;

×

71

      if (comparefn(pKey, keyList, firstPos) < 0) return firstPos - 1;

×

73

      numOfRows = lastPos - firstPos + 1;

×

74

      midPos = (numOfRows >> 1) + firstPos;

×

76

      if (comparefn(pKey, keyList, midPos) < 0) {

×

77

        lastPos = midPos - 1;

×

78

      } else if (comparefn(pKey, keyList, midPos) > 0) {

×

79

        firstPos = midPos + 1;

×

81

        break;

×

88

      if (comparefn(pKey, keyList, firstPos) <= 0) return firstPos;

×

89

      if (comparefn(pKey, keyList, lastPos) == 0) return lastPos;

×

91

      if (comparefn(pKey, keyList, lastPos) > 0) {

×

92

        lastPos = lastPos + 1;

×

93

        if (lastPos >= num)

×

94

          return -1;

×

96

          return lastPos;

×

99

      numOfRows = lastPos - firstPos + 1;

×

100

      midPos = (numOfRows >> 1) + firstPos;

×

102

      if (comparefn(pKey, keyList, midPos) < 0) {

×

103

        lastPos = midPos - 1;

×

104

      } else if (comparefn(pKey, keyList, midPos) > 0) {

×

105

        firstPos = midPos + 1;

×

107

        break;

×

112

  return midPos;

×

115

static int32_t comparePullWinKey(void* pKey, void* data, int32_t index) {

×

116

  SArray*          res = (SArray*)data;

×

117

  SPullWindowInfo* pos = taosArrayGet(res, index);

×

118

  SPullWindowInfo* pData = (SPullWindowInfo*)pKey;

×

119

  if (pData->groupId > pos->groupId) {

×

120

    return 1;

×

121

  } else if (pData->groupId < pos->groupId) {

×

122

    return -1;

×

125

  if (pData->window.skey > pos->window.ekey) {

×

126

    return 1;

×

127

  } else if (pData->window.ekey < pos->window.skey) {

×

128

    return -1;

×

130

  return 0;

×

133

static int32_t savePullWindow(SPullWindowInfo* pPullInfo, SArray* pPullWins) {

×

134

  int32_t size = taosArrayGetSize(pPullWins);

×

135

  int32_t index = binarySearchCom(pPullWins, size, pPullInfo, TSDB_ORDER_DESC, comparePullWinKey);

×

136

  if (index == -1) {

×

137

    index = 0;

×

139

    int32_t code = comparePullWinKey(pPullInfo, pPullWins, index);

×

140

    if (code == 0) {

×

141

      SPullWindowInfo* pos = taosArrayGet(pPullWins, index);

×

142

      pos->window.skey = TMIN(pos->window.skey, pPullInfo->window.skey);

×

143

      pos->window.ekey = TMAX(pos->window.ekey, pPullInfo->window.ekey);

×

144

      pos->calWin.skey = TMIN(pos->calWin.skey, pPullInfo->calWin.skey);

×

145

      pos->calWin.ekey = TMAX(pos->calWin.ekey, pPullInfo->calWin.ekey);

×

146

      return TSDB_CODE_SUCCESS;

×

147

    } else if (code > 0) {

×

148

      index++;

×

151

  if (taosArrayInsert(pPullWins, index, pPullInfo) == NULL) {

×

152

    return terrno;

×

154

  return TSDB_CODE_SUCCESS;

×

157

int32_t saveResult(SResultWindowInfo winInfo, SSHashObj* pStUpdated) {

×

158

  if (tSimpleHashGetSize(pStUpdated) > MAX_STREAM_HISTORY_RESULT) {

×

159

    qError("%s failed at line %d since too many history result. ", __func__, __LINE__);

×

160

    return TSDB_CODE_STREAM_INTERNAL_ERROR;

×

162

  winInfo.sessionWin.win.ekey = winInfo.sessionWin.win.skey;

×

163

  return tSimpleHashPut(pStUpdated, &winInfo.sessionWin, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo));

×

166

int32_t saveWinResult(SWinKey* pKey, SRowBuffPos* pPos, SSHashObj* pUpdatedMap) {

×

167

  if (tSimpleHashGetSize(pUpdatedMap) > MAX_STREAM_HISTORY_RESULT) {

×

168

    qError("%s failed at line %d since too many history result. ", __func__, __LINE__);

×

169

    return TSDB_CODE_STREAM_INTERNAL_ERROR;

×

171

  return tSimpleHashPut(pUpdatedMap, pKey, sizeof(SWinKey), &pPos, POINTER_BYTES);

×

174

static int32_t saveWinResultInfo(TSKEY ts, uint64_t groupId, SRowBuffPos* pPos, SSHashObj* pUpdatedMap) {

×

175

  SWinKey key = {.ts = ts, .groupId = groupId};

×

176

  return saveWinResult(&key, pPos, pUpdatedMap);

×

179

static void removeResults(SArray* pWins, SSHashObj* pUpdatedMap) {

×

180

  int32_t size = taosArrayGetSize(pWins);

×

181

  for (int32_t i = 0; i < size; i++) {

×

182

    SWinKey* pW = taosArrayGet(pWins, i);

×

183

    void*    tmp = tSimpleHashGet(pUpdatedMap, pW, sizeof(SWinKey));

×

184

    if (tmp) {

×

185

      void* value = *(void**)tmp;

×

186

      taosMemoryFree(value);

×

187

      int32_t tmpRes = tSimpleHashRemove(pUpdatedMap, pW, sizeof(SWinKey));

×

188

      qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

191

×

193

static int32_t compareWinKey(void* pKey, void* data, int32_t index) {

×

194

  void* pDataPos = taosArrayGet((SArray*)data, index);

×

195

  return winKeyCmprImpl(pKey, pDataPos);

×

198

void removeDeleteResults(SSHashObj* pUpdatedMap, SArray* pDelWins) {

×

199

  taosArraySort(pDelWins, winKeyCmprImpl);

×

200

  taosArrayRemoveDuplicate(pDelWins, winKeyCmprImpl, NULL);

×

201

  int32_t delSize = taosArrayGetSize(pDelWins);

×

202

  if (tSimpleHashGetSize(pUpdatedMap) == 0 || delSize == 0) {

×

203

    return;

×

205

  void*   pIte = NULL;

×

206

  int32_t iter = 0;

×

207

  while ((pIte = tSimpleHashIterate(pUpdatedMap, pIte, &iter)) != NULL) {

×

208

    SWinKey* pResKey = tSimpleHashGetKey(pIte, NULL);

×

209

    int32_t  index = binarySearchCom(pDelWins, delSize, pResKey, TSDB_ORDER_DESC, compareWinKey);

×

210

    if (index >= 0 && 0 == compareWinKey(pResKey, pDelWins, index)) {

×

211

      taosArrayRemove(pDelWins, index);

×

212

      delSize = taosArrayGetSize(pDelWins);

×

217

bool isOverdue(TSKEY ekey, STimeWindowAggSupp* pTwSup) {

×

218

  return pTwSup->maxTs != INT64_MIN && ekey < pTwSup->maxTs - pTwSup->waterMark;

×

221

bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pTwSup) { return isOverdue(pWin->ekey, pTwSup); }

×

223

static void doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId) {

×

224

  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;

×

226

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

227

  SWinKey                      key = {.ts = ts, .groupId = groupId};

×

228

  int32_t                      tmpRes = tSimpleHashRemove(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey));

×

229

  qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

230

  pAPI->stateStore.streamStateDel(pInfo->pState, &key);

×

231

×

233

static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; }

×

235

static void doDeleteWindowByGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock) {

×

236

  SStorageAPI*                 pAPI = &pOperator->pTaskInfo->storageAPI;

×

237

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

239

  SColumnInfoData* pGpIdCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);

×

240

  uint64_t* pGroupIdData = (uint64_t*)pGpIdCol->pData;

×

241

  for (int32_t i = 0; i < pBlock->info.rows; i++) {

×

242

    uint64_t groupId = pGroupIdData[i];

×

243

    void*   pIte = NULL;

×

244

    int32_t iter = 0;

×

245

    while ((pIte = tSimpleHashIterate(pInfo->aggSup.pResultRowHashTable, pIte, &iter)) != NULL) {

×

246

      size_t keyLen = 0;

×

247

      SWinKey* pKey = tSimpleHashGetKey(pIte, &keyLen);

×

248

      if (pKey->groupId == groupId) {

×

249

        int32_t tmpRes = tSimpleHashIterateRemove(pInfo->aggSup.pResultRowHashTable, pKey, keyLen, &pIte, &iter);

×

250

        qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

254

    pAPI->stateStore.streamStateDelByGroupId(pInfo->pState, groupId);

×

256

×

258

static int32_t doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDataBlock* pBlock, SArray* pUpWins,

×

260

  int32_t                      code = TSDB_CODE_SUCCESS;

×

261

  int32_t                      lino = 0;

×

262

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

263

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

264

  SColumnInfoData*             pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);

×

265

  TSKEY*                       startTsCols = (TSKEY*)pStartTsCol->pData;

×

266

  SColumnInfoData*             pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);

×

267

  TSKEY*                       endTsCols = (TSKEY*)pEndTsCol->pData;

×

268

  SColumnInfoData*             pCalStTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);

×

269

  TSKEY*                       calStTsCols = (TSKEY*)pCalStTsCol->pData;

×

270

  SColumnInfoData*             pCalEnTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);

×

271

  TSKEY*                       calEnTsCols = (TSKEY*)pCalEnTsCol->pData;

×

272

  SColumnInfoData*             pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);

×

273

  uint64_t*                    pGpDatas = (uint64_t*)pGpCol->pData;

×

274

  for (int32_t i = 0; i < pBlock->info.rows; i++) {

×

275

    SResultRowInfo dumyInfo = {0};

×

276

    dumyInfo.cur.pageId = -1;

×

278

    STimeWindow win = {0};

×

279

    if (IS_FINAL_INTERVAL_OP(pOperator) || IS_MID_INTERVAL_OP(pOperator)) {

×

280

      win.skey = startTsCols[i];

×

281

      win.ekey = endTsCols[i];

×

283

      win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC);

×

287

      if (!inCalSlidingWindow(pInterval, &win, calStTsCols[i], calEnTsCols[i], pBlock->info.type)) {

×

288

        getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC);

×

289

        continue;

×

291

      uint64_t winGpId = pGpDatas[i];

×

292

      SWinKey  winRes = {.ts = win.skey, .groupId = winGpId};

×

293

      void*    chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey));

×

294

      if (chIds) {

×

295

        int32_t childId = getChildIndex(pBlock);

×

296

        if (pInvalidWins) {

×

297

          qDebug("===stream===save invalid delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d", winRes.ts,

×

299

          code = taosHashPut(pInvalidWins, &winRes, sizeof(SWinKey), NULL, 0);

×

300

          QUERY_CHECK_CODE(code, lino, _end);

×

303

        SArray* chArray = *(void**)chIds;

×

304

        int32_t index = taosArraySearchIdx(chArray, &childId, compareInt32Val, TD_EQ);

×

305

        if (index != -1) {

×

306

          qDebug("===stream===try push delete window:%" PRId64 ",groupId:%" PRId64 ",chId:%d ,continue", win.skey,

×

308

          getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC);

×

309

          continue;

×

312

      doDeleteWindow(pOperator, win.skey, winGpId);

×

313

      if (pUpWins) {

×

314

        void* tmp = taosArrayPush(pUpWins, &winRes);

×

315

        if (!tmp) {

×

316

          code = terrno;

×

317

          QUERY_CHECK_CODE(code, lino, _end);

×

320

      if (pUpdatedMap) {

×

321

        int32_t tmpRes = tSimpleHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey));

×

322

        qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

324

      getNextTimeWindow(pInterval, &win, TSDB_ORDER_ASC);

×

325

    } while (win.ekey <= endTsCols[i]);

×

327

_end:

×

328

  if (code != TSDB_CODE_SUCCESS) {

×

329

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

331

  return code;

×

334

static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SSHashObj* resWins) {

×

335

  void*   pIte = NULL;

×

336

  int32_t iter = 0;

×

337

  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {

×

338

    SWinKey*     pKey = tSimpleHashGetKey(pIte, NULL);

×

339

    uint64_t     groupId = pKey->groupId;

×

340

    TSKEY        ts = pKey->ts;

×

341

    SRowBuffPos* pPos = *(SRowBuffPos**)pIte;

×

342

    if (!pPos->beUpdated) {

×

343

      continue;

×

345

    pPos->beUpdated = false;

×

346

    int32_t code = saveWinResultInfo(ts, groupId, pPos, resWins);

×

347

    if (code != TSDB_CODE_SUCCESS) {

×

348

      return code;

×

351

  return TSDB_CODE_SUCCESS;

×

354

static int32_t closeStreamIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SInterval* pInterval,

×

357

  qDebug("===stream===close interval window");

×

358

  int32_t                      code = TSDB_CODE_SUCCESS;

×

359

  int32_t                      lino = 0;

×

360

  void*                        pIte = NULL;

×

361

  int32_t                      iter = 0;

×

362

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

363

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

364

  int32_t                      delSize = taosArrayGetSize(pDelWins);

×

365

  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {

×

366

    void*    key = tSimpleHashGetKey(pIte, NULL);

×

367

    SWinKey* pWinKey = (SWinKey*)key;

×

368

    if (delSize > 0) {

×

369

      int32_t index = binarySearchCom(pDelWins, delSize, pWinKey, TSDB_ORDER_DESC, compareWinKey);

×

370

      if (index >= 0 && 0 == compareWinKey(pWinKey, pDelWins, index)) {

×

371

        taosArrayRemove(pDelWins, index);

×

372

        delSize = taosArrayGetSize(pDelWins);

×

376

    void*       chIds = taosHashGet(pPullDataMap, pWinKey, sizeof(SWinKey));

×

377

    STimeWindow win = {

×

378

        .skey = pWinKey->ts,

×

379

        .ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision, NULL) - 1,

×

381

    if (isCloseWindow(&win, pTwSup)) {

×

382

      if (chIds && pPullDataMap) {

×

383

        SArray* chAy = *(SArray**)chIds;

×

384

        int32_t size = taosArrayGetSize(chAy);

×

385

        qDebug("===stream===window %" PRId64 " wait child size:%d", pWinKey->ts, size);

×

386

        for (int32_t i = 0; i < size; i++) {

×

387

          qDebug("===stream===window %" PRId64 " wait child id:%d", pWinKey->ts, *(int32_t*)taosArrayGet(chAy, i));

×

389

        continue;

×

390

      } else if (pPullDataMap) {

×

391

        qDebug("===stream===close window %" PRId64, pWinKey->ts);

×

394

      if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

395

        code = saveWinResult(pWinKey, *(SRowBuffPos**)pIte, closeWins);

×

396

        QUERY_CHECK_CODE(code, lino, _end);

×

398

      int32_t tmpRes = tSimpleHashIterateRemove(pHashMap, pWinKey, sizeof(SWinKey), &pIte, &iter);

×

399

      qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

403

_end:

×

404

  if (code != TSDB_CODE_SUCCESS) {

×

405

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

407

  return code;

×

410

STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) {

×

411

  STimeWindow w = {.skey = ts, .ekey = INT64_MAX};

×

412

  w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision, NULL) - 1;

×

413

  return w;

×

416

static void doBuildDeleteResult(SStreamIntervalOperatorInfo* pInfo, SArray* pWins, int32_t* index,

×

418

  doBuildDeleteResultImpl(&pInfo->stateStore, pInfo->pState, pWins, index, pBlock);

×

419

×

421

void doBuildDeleteResultImpl(SStateStore* pAPI, SStreamState* pState, SArray* pWins, int32_t* index,

×

423

  int32_t code = TSDB_CODE_SUCCESS;

×

424

  int32_t lino = 0;

×

425

  blockDataCleanup(pBlock);

×

426

  int32_t size = taosArrayGetSize(pWins);

×

427

  if (*index == size) {

×

428

    *index = 0;

×

429

    taosArrayClear(pWins);

×

430

    goto _end;

×

432

  code = blockDataEnsureCapacity(pBlock, size - *index);

×

433

  QUERY_CHECK_CODE(code, lino, _end);

×

435

  uint64_t uid = 0;

×

436

  for (int32_t i = *index; i < size; i++) {

×

437

    SWinKey* pWin = taosArrayGet(pWins, i);

×

438

    void*    tbname = NULL;

×

439

    int32_t  winCode = TSDB_CODE_SUCCESS;

×

440

    code = pAPI->streamStateGetParName(pState, pWin->groupId, &tbname, false, &winCode);

×

441

    QUERY_CHECK_CODE(code, lino, _end);

×

443

    if (winCode != TSDB_CODE_SUCCESS) {

×

444

      code = appendDataToSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, NULL);

×

445

      QUERY_CHECK_CODE(code, lino, _end);

×

447

      QUERY_CHECK_CONDITION((tbname), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);

×

449

      STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));

×

450

      code = appendDataToSpecialBlock(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId, parTbName);

×

451

      QUERY_CHECK_CODE(code, lino, _end);

×

453

    pAPI->streamStateFreeVal(tbname);

×

454

    (*index)++;

×

457

_end:

×

458

  if (code != TSDB_CODE_SUCCESS) {

×

459

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

461

×

463

void destroyFlusedPos(void* pRes) {

×

464

  SRowBuffPos* pPos = (SRowBuffPos*)pRes;

×

465

  if (pPos->needFree && !pPos->pRowBuff) {

×

466

    taosMemoryFreeClear(pPos->pKey);

×

467

    taosMemoryFree(pPos);

×

469

×

471

void destroyFlusedppPos(void* ppRes) {

×

472

  void* pRes = *(void**)ppRes;

×

473

  destroyFlusedPos(pRes);

×

474

×

476

void clearGroupResInfo(SGroupResInfo* pGroupResInfo) {

×

477

  int32_t size = taosArrayGetSize(pGroupResInfo->pRows);

×

478

  if (pGroupResInfo->index >= 0 && pGroupResInfo->index < size) {

×

479

    for (int32_t i = pGroupResInfo->index; i < size; i++) {

×

480

      void* pPos = taosArrayGetP(pGroupResInfo->pRows, i);

×

481

      destroyFlusedPos(pPos);

×

484

  pGroupResInfo->freeItem = false;

×

485

  taosArrayDestroy(pGroupResInfo->pRows);

×

486

  pGroupResInfo->pRows = NULL;

×

487

  pGroupResInfo->index = 0;

×

488

  pGroupResInfo->delIndex = 0;

×

489

×

491

void destroyStreamFinalIntervalOperatorInfo(void* param) {

×

492

  if (param == NULL) {

×

493

    return;

×

495

  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)param;

×

496

  cleanupBasicInfo(&pInfo->binfo);

×

497

  if (pInfo->pOperator) {

×

498

    cleanupResultInfo(pInfo->pOperator->pTaskInfo, &pInfo->pOperator->exprSupp, &pInfo->groupResInfo, &pInfo->aggSup,

×

500

    pInfo->pOperator = NULL;

×

503

  destroyStreamBasicInfo(&pInfo->basic);

×

504

  cleanupAggSup(&pInfo->aggSup);

×

505

  clearGroupResInfo(&pInfo->groupResInfo);

×

506

  taosArrayDestroyP(pInfo->pUpdated, destroyFlusedPos);

×

507

  pInfo->pUpdated = NULL;

×

510

  void* pIte = NULL;

×

511

  while ((pIte = taosHashIterate(pInfo->pPullDataMap, pIte)) != NULL) {

×

512

    taosArrayDestroy(*(void**)pIte);

×

514

  taosHashCleanup(pInfo->pPullDataMap);

×

515

  taosHashCleanup(pInfo->pFinalPullDataMap);

×

516

  taosArrayDestroy(pInfo->pPullWins);

×

517

  blockDataDestroy(pInfo->pPullDataRes);

×

518

  taosArrayDestroy(pInfo->pDelWins);

×

519

  blockDataDestroy(pInfo->pDelRes);

×

520

  blockDataDestroy(pInfo->pMidRetriveRes);

×

521

  blockDataDestroy(pInfo->pMidPulloverRes);

×

522

  if (pInfo->pUpdatedMap != NULL) {

×

524

    tSimpleHashSetFreeFp(pInfo->pUpdatedMap, destroyFlusedppPos);

×

525

    tSimpleHashCleanup(pInfo->pUpdatedMap);

×

526

    pInfo->pUpdatedMap = NULL;

×

529

  if (pInfo->stateStore.streamFileStateDestroy != NULL) {

×

530

    pInfo->stateStore.streamFileStateDestroy(pInfo->pState->pFileState);

×

532

  taosArrayDestroy(pInfo->pMidPullDatas);

×

534

  if (pInfo->pState != NULL && pInfo->pState->dump == 1) {

×

535

    taosMemoryFreeClear(pInfo->pState->pTdbState->pOwner);

×

536

    taosMemoryFreeClear(pInfo->pState->pTdbState);

×

538

  taosMemoryFreeClear(pInfo->pState);

×

540

  nodesDestroyNode((SNode*)pInfo->pPhyNode);

×

541

  colDataDestroy(&pInfo->twAggSup.timeWindowData);

×

542

  cleanupExprSupp(&pInfo->scalarSupp);

×

543

  tSimpleHashCleanup(pInfo->pDeletedMap);

×

545

  blockDataDestroy(pInfo->pCheckpointRes);

×

547

  taosMemoryFreeClear(param);

×

561

void reloadFromDownStream(SOperatorInfo* downstream, SStreamIntervalOperatorInfo* pInfo) {

×

562

  SStateStore* pAPI = &downstream->pTaskInfo->storageAPI.stateStore;

×

564

  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {

×

565

    reloadFromDownStream(downstream->pDownstream[0], pInfo);

×

566

    return;

×

569

  SStreamScanInfo* pScanInfo = downstream->info;

×

570

  pInfo->pUpdateInfo = pScanInfo->pUpdateInfo;

×

573

bool hasSrcPrimaryKeyCol(SSteamOpBasicInfo* pInfo) { return pInfo->primaryPkIndex != -1; }

×

575

int32_t initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SStreamIntervalOperatorInfo* pInfo,

×

577

  int32_t        code = TSDB_CODE_SUCCESS;

×

578

  int32_t        lino = 0;

×

579

  SStateStore*   pAPI = &downstream->pTaskInfo->storageAPI.stateStore;

×

580

  SExecTaskInfo* pTaskInfo = downstream->pTaskInfo;

×

582

  if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) {

×

583

    SStreamPartitionOperatorInfo* pScanInfo = downstream->info;

×

584

    pBasic->primaryPkIndex = pScanInfo->basic.primaryPkIndex;

×

587

  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {

×

588

    return initIntervalDownStream(downstream->pDownstream[0], type, pInfo, pBasic);

×

591

  SStreamScanInfo* pScanInfo = downstream->info;

×

592

  pScanInfo->windowSup.parentType = type;

×

593

  pScanInfo->windowSup.pIntervalAggSup = &pInfo->aggSup;

×

594

  if (!pScanInfo->pUpdateInfo) {

×

595

    code = pAPI->updateInfoInitP(&pInfo->interval, pInfo->twAggSup.waterMark, pScanInfo->igCheckUpdate,

×

596

                                 pScanInfo->pkColType, pScanInfo->pkColLen, &pScanInfo->pUpdateInfo);

×

597

    QUERY_CHECK_CODE(code, lino, _end);

×

600

  pScanInfo->interval = pInfo->interval;

×

601

  pScanInfo->twAggSup = pInfo->twAggSup;

×

602

  pScanInfo->pState = pInfo->pState;

×

603

  pInfo->pUpdateInfo = pScanInfo->pUpdateInfo;

×

604

  if (!hasSrcPrimaryKeyCol(pBasic)) {

×

605

    pBasic->primaryPkIndex = pScanInfo->basic.primaryPkIndex;

×

608

_end:

×

609

  if (code != TSDB_CODE_SUCCESS) {

×

610

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

612

  return code;

×

615

int32_t compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput,

×

617

  int32_t code = TSDB_CODE_SUCCESS;

×

618

  int32_t lino = 0;

×

619

  for (int32_t k = 0; k < numOfOutput; ++k) {

×

620

    if (fmIsWindowPseudoColumnFunc(pDestCtx[k].functionId)) {

×

621

      if (!pTimeWindowData) {

×

622

        continue;

×

625

      SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pDestCtx[k]);

×

626

      char*                p = GET_ROWCELL_INTERBUF(pEntryInfo);

×

627

      SColumnInfoData      idata = {0};

×

628

      idata.info.type = TSDB_DATA_TYPE_BIGINT;

×

629

      idata.info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes;

×

630

      idata.pData = p;

×

632

      SScalarParam out = {.columnData = &idata};

×

633

      SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData};

×

634

      code = pDestCtx[k].sfp.process(&tw, 1, &out);

×

635

      QUERY_CHECK_CODE(code, lino, _end);

×

637

      pEntryInfo->numOfRes = 1;

×

638

    } else if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) {

×

639

      code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]);

×

640

      QUERY_CHECK_CODE(code, lino, _end);

×

641

    } else if (pDestCtx[k].fpSet.combine == NULL) {

×

642

      char* funName = fmGetFuncName(pDestCtx[k].functionId);

×

643

      qError("%s error, combine funcion for %s is not implemented", GET_TASKID(pTaskInfo), funName);

×

644

      taosMemoryFreeClear(funName);

×

645

      code = TSDB_CODE_FAILED;

×

646

      QUERY_CHECK_CODE(code, lino, _end);

×

650

_end:

×

651

  if (code != TSDB_CODE_SUCCESS) {

×

652

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

654

  return code;

×

657

bool hasIntervalWindow(void* pState, SWinKey* pKey, SStateStore* pStore) {

×

658

  return pStore->streamStateCheck(pState, pKey);

×

661

int32_t setIntervalOutputBuf(void* pState, STimeWindow* win, SRowBuffPos** pResult, int64_t groupId,

×

664

  int32_t code = TSDB_CODE_SUCCESS;

×

665

  int32_t lino = 0;

×

666

  SWinKey key = {.ts = win->skey, .groupId = groupId};

×

667

  char*   value = NULL;

×

668

  int32_t size = pAggSup->resultRowSize;

×

670

  code = pStore->streamStateAddIfNotExist(pState, &key, (void**)&value, &size, pWinCode);

×

671

  QUERY_CHECK_CODE(code, lino, _end);

×

673

  *pResult = (SRowBuffPos*)value;

×

674

  SResultRow* res = (SResultRow*)((*pResult)->pRowBuff);

×

677

  res->win = (*win);

×

678

  code = setResultRowInitCtx(res, pCtx, numOfOutput, rowEntryInfoOffset);

×

679

  QUERY_CHECK_CODE(code, lino, _end);

×

681

_end:

×

682

  if (code != TSDB_CODE_SUCCESS) {

×

683

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

685

  return code;

×

688

bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, STimeWindowAggSupp* pTwSup,

×

690

  if (pTwSup->maxTs != INT64_MIN && pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark) {

×

691

    SWinKey key = {.ts = pWin->skey, .groupId = groupId};

×

692

    if (!hasIntervalWindow(pState, &key, pStore)) {

×

693

      return true;

×

695

    return false;

×

697

  return false;

×

700

int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols, int32_t startPos, TSKEY eKey,

×

703

      getNumOfRowsInTimeWindow(pBlockInfo, tsCols, startPos, eKey, binarySearchForKey, NULL, TSDB_ORDER_ASC);

×

704

  int32_t prevEndPos = forwardRows - 1 + startPos;

×

705

  return getNextQualifiedWindow(pInterval, pNextWin, pBlockInfo, tsCols, prevEndPos, TSDB_ORDER_ASC);

×

708

int32_t addPullWindow(SHashObj* pMap, SWinKey* pWinRes, int32_t size) {

×

709

  int32_t code = TSDB_CODE_SUCCESS;

×

710

  int32_t lino = 0;

×

711

  SArray* childIds = taosArrayInit(8, sizeof(int32_t));

×

712

  QUERY_CHECK_NULL(childIds, code, lino, _end, terrno);

×

713

  for (int32_t i = 0; i < size; i++) {

×

714

    void* tmp = taosArrayPush(childIds, &i);

×

715

    if (!tmp) {

×

716

      code = terrno;

×

717

      QUERY_CHECK_CODE(code, lino, _end);

×

720

  code = taosHashPut(pMap, pWinRes, sizeof(SWinKey), &childIds, sizeof(void*));

×

721

  QUERY_CHECK_CODE(code, lino, _end);

×

722

_end:

×

723

  if (code != TSDB_CODE_SUCCESS) {

×

724

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

726

  return code;

×

729

static void clearStreamIntervalOperator(SStreamIntervalOperatorInfo* pInfo) {

×

730

  tSimpleHashClear(pInfo->aggSup.pResultRowHashTable);

×

731

  clearDiskbasedBuf(pInfo->aggSup.pResultBuf);

×

732

  initResultRowInfo(&pInfo->binfo.resultRowInfo);

×

733

  pInfo->aggSup.currentPageId = -1;

×

734

  pInfo->stateStore.streamStateClear(pInfo->pState);

×

735

×

737

static void clearSpecialDataBlock(SSDataBlock* pBlock) {

×

738

  if (pBlock->info.rows <= 0) {

×

739

    return;

×

741

  blockDataCleanup(pBlock);

×

744

static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pBlock) {

×

745

  int32_t code = TSDB_CODE_SUCCESS;

×

746

  int32_t lino = 0;

×

747

  clearSpecialDataBlock(pBlock);

×

748

  int32_t size = taosArrayGetSize(array);

×

749

  if (size - (*pIndex) == 0) {

×

750

    goto _end;

×

752

  code = blockDataEnsureCapacity(pBlock, size - (*pIndex));

×

753

  QUERY_CHECK_CODE(code, lino, _end);

×

755

  SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);

×

756

  SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);

×

757

  SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);

×

758

  SColumnInfoData* pCalStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);

×

759

  SColumnInfoData* pCalEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);

×

760

  SColumnInfoData* pTbName = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);

×

761

  SColumnInfoData* pPrimaryKey = NULL;

×

762

  if (taosArrayGetSize(pBlock->pDataBlock) > PRIMARY_KEY_COLUMN_INDEX) {

×

763

    pPrimaryKey = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, PRIMARY_KEY_COLUMN_INDEX);

×

765

  for (; (*pIndex) < size; (*pIndex)++) {

×

766

    SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex));

×

767

    code = colDataSetVal(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false);

×

768

    QUERY_CHECK_CODE(code, lino, _end);

×

770

    code = colDataSetVal(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false);

×

771

    QUERY_CHECK_CODE(code, lino, _end);

×

773

    code = colDataSetVal(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false);

×

774

    QUERY_CHECK_CODE(code, lino, _end);

×

776

    code = colDataSetVal(pCalStartTs, pBlock->info.rows, (const char*)&pWin->calWin.skey, false);

×

777

    QUERY_CHECK_CODE(code, lino, _end);

×

779

    code = colDataSetVal(pCalEndTs, pBlock->info.rows, (const char*)&pWin->calWin.ekey, false);

×

780

    QUERY_CHECK_CODE(code, lino, _end);

×

782

    colDataSetNULL(pTbName, pBlock->info.rows);

×

783

    if (pPrimaryKey != NULL) {

×

784

      colDataSetNULL(pPrimaryKey, pBlock->info.rows);

×

787

    pBlock->info.rows++;

×

789

  if ((*pIndex) == size) {

×

790

    *pIndex = 0;

×

791

    taosArrayClear(array);

×

793

  code = blockDataUpdateTsWindow(pBlock, 0);

×

794

  QUERY_CHECK_CODE(code, lino, _end);

×

796

_end:

×

797

  if (code != TSDB_CODE_SUCCESS) {

×

798

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

800

×

802

static int32_t processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SHashObj* pFinalMap, SInterval* pInterval,

×

804

  int32_t                      code = TSDB_CODE_SUCCESS;

×

805

  int32_t                      lino = 0;

×

806

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

807

  SColumnInfoData*             pStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);

×

808

  TSKEY*                       tsData = (TSKEY*)pStartCol->pData;

×

809

  SColumnInfoData*             pEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);

×

810

  TSKEY*                       tsEndData = (TSKEY*)pEndCol->pData;

×

811

  SColumnInfoData*             pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);

×

812

  uint64_t*                    groupIdData = (uint64_t*)pGroupCol->pData;

×

813

  int32_t                      chId = getChildIndex(pBlock);

×

814

  bool                         res = false;

×

815

  for (int32_t i = 0; i < pBlock->info.rows; i++) {

×

816

    TSKEY winTs = tsData[i];

×

817

    while (winTs <= tsEndData[i]) {

×

818

      SWinKey winRes = {.ts = winTs, .groupId = groupIdData[i]};

×

819

      void*   chIds = taosHashGet(pMap, &winRes, sizeof(SWinKey));

×

820

      if (chIds) {

×

821

        SArray* chArray = *(SArray**)chIds;

×

822

        int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);

×

823

        if (index != -1) {

×

824

          qDebug("===stream===retrive window %" PRId64 " delete child id %d", winRes.ts, chId);

×

825

          taosArrayRemove(chArray, index);

×

826

          if (taosArrayGetSize(chArray) == 0) {

×

828

            taosArrayDestroy(chArray);

×

829

            int32_t tmpRes = taosHashRemove(pMap, &winRes, sizeof(SWinKey));

×

830

            qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

831

            res = true;

×

832

            qDebug("===stream===retrive pull data over.window %" PRId64, winRes.ts);

×

834

            void* pFinalCh = taosHashGet(pFinalMap, &winRes, sizeof(SWinKey));

×

835

            if (pFinalCh) {

×

836

              int32_t tmpRes = taosHashRemove(pFinalMap, &winRes, sizeof(SWinKey));

×

837

              qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

838

              doDeleteWindow(pOperator, winRes.ts, winRes.groupId);

×

839

              STimeWindow     nextWin = getFinalTimeWindow(winRes.ts, pInterval);

×

840

              SPullWindowInfo pull = {.window = nextWin,

×

841

                                      .groupId = winRes.groupId,

×

842

                                      .calWin.skey = nextWin.skey,

×

843

                                      .calWin.ekey = nextWin.skey};

×

845

              qDebug("===stream===prepare final retrive for delete window:%" PRId64 ",groupId:%" PRId64 ", size:%d",

×

847

              if (IS_MID_INTERVAL_OP(pOperator)) {

×

848

                SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info;

×

850

                void* tmp = taosArrayPush(pInfo->pMidPullDatas, &winRes);

×

851

                if (!tmp) {

×

852

                  code = terrno;

×

853

                  QUERY_CHECK_CODE(code, lino, _end);

×

855

              } else if (savePullWindow(&pull, pPullWins) == TSDB_CODE_SUCCESS) {

×

856

                void* tmp = taosArrayPush(pInfo->pDelWins, &winRes);

×

857

                if (!tmp) {

×

858

                  code = terrno;

×

859

                  QUERY_CHECK_CODE(code, lino, _end);

×

862

                code = addPullWindow(pMap, &winRes, numOfCh);

×

863

                QUERY_CHECK_CODE(code, lino, _end);

×

865

                if (pInfo->destHasPrimaryKey) {

×

866

                  code = tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0);

×

867

                  QUERY_CHECK_CODE(code, lino, _end);

×

869

                qDebug("===stream===prepare final retrive for delete %" PRId64 ", size:%d", winRes.ts, numOfCh);

×

875

      winTs = taosTimeAdd(winTs, pInterval->sliding, pInterval->slidingUnit, pInterval->precision, NULL);

×

878

  if (pBeOver) {

×

879

    *pBeOver = res;

×

882

_end:

×

883

  if (code != TSDB_CODE_SUCCESS) {

×

884

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

886

  return code;

×

889

static int32_t addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo, int32_t childId) {

×

890

  int32_t code = TSDB_CODE_SUCCESS;

×

891

  int32_t lino = 0;

×

892

  int32_t size = taosArrayGetSize(wins);

×

893

  for (int32_t i = 0; i < size; i++) {

×

894

    SWinKey*    winKey = taosArrayGet(wins, i);

×

895

    STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval);

×

896

    void*       chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey));

×

897

    if (!chIds) {

×

898

      SPullWindowInfo pull = {

×

899

          .window = nextWin, .groupId = winKey->groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey};

×

901

      if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) {

×

902

        code = addPullWindow(pInfo->pPullDataMap, winKey, pInfo->numOfChild);

×

903

        QUERY_CHECK_CODE(code, lino, _end);

×

905

        if (pInfo->destHasPrimaryKey) {

×

906

          code = tSimpleHashPut(pInfo->pDeletedMap, winKey, sizeof(SWinKey), NULL, 0);

×

907

          QUERY_CHECK_CODE(code, lino, _end);

×

909

        qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, pInfo->numOfChild);

×

912

      SArray* chArray = *(void**)chIds;

×

913

      int32_t index = taosArraySearchIdx(chArray, &childId, compareInt32Val, TD_EQ);

×

914

      qDebug("===stream===check final retrive %" PRId64 ",chid:%d", winKey->ts, index);

×

915

      if (index == -1) {

×

916

        qDebug("===stream===add final retrive %" PRId64, winKey->ts);

×

917

        code = taosHashPut(pInfo->pFinalPullDataMap, winKey, sizeof(SWinKey), NULL, 0);

×

918

        QUERY_CHECK_CODE(code, lino, _end);

×

923

_end:

×

924

  if (code != TSDB_CODE_SUCCESS) {

×

925

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

927

  return code;

×

930

static void clearFunctionContext(SExprSupp* pSup) {

×

931

  for (int32_t i = 0; i < pSup->numOfExprs; i++) {

×

932

    pSup->pCtx[i].saveHandle.currentPage = -1;

×

934

×

936

int32_t getOutputBuf(void* pState, SRowBuffPos* pPos, SResultRow** pResult, SStateStore* pStore) {

×

937

  return pStore->streamStateGetByPos(pState, pPos, (void**)pResult);

×

940

void buildDataBlockFromGroupRes(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup,

×

942

  int32_t         code = TSDB_CODE_SUCCESS;

×

943

  int32_t         lino = 0;

×

944

  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;

×

945

  SStorageAPI*    pAPI = &pOperator->pTaskInfo->storageAPI;

×

946

  SExprInfo*      pExprInfo = pSup->pExprInfo;

×

947

  int32_t         numOfExprs = pSup->numOfExprs;

×

948

  int32_t*        rowEntryOffset = pSup->rowEntryInfoOffset;

×

949

  SqlFunctionCtx* pCtx = pSup->pCtx;

×

951

  int32_t numOfRows = getNumOfTotalRes(pGroupResInfo);

×

953

  for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) {

×

954

    SRowBuffPos* pPos = *(SRowBuffPos**)taosArrayGet(pGroupResInfo->pRows, i);

×

955

    SResultRow*  pRow = NULL;

×

956

    code = getOutputBuf(pState, pPos, &pRow, &pAPI->stateStore);

×

957

    QUERY_CHECK_CODE(code, lino, _end);

×

958

    uint64_t groupId = ((SWinKey*)pPos->pKey)->groupId;

×

959

    doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset);

×

961

    if (pRow->numOfRows == 0) {

×

962

      pGroupResInfo->index += 1;

×

963

      continue;

×

965

    if (pBlock->info.id.groupId == 0) {

×

966

      pBlock->info.id.groupId = groupId;

×

967

      void*   tbname = NULL;

×

968

      int32_t winCode = TSDB_CODE_SUCCESS;

×

969

      code = pAPI->stateStore.streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname,

×

971

      QUERY_CHECK_CODE(code, lino, _end);

×

972

      if (winCode != TSDB_CODE_SUCCESS) {

×

973

        pBlock->info.parTbName[0] = 0;

×

975

        memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);

×

977

      pAPI->stateStore.streamStateFreeVal(tbname);

×

980

      if (pBlock->info.id.groupId != groupId) {

×

981

        break;

×

985

    if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) {

×

986

      break;

×

988

    pGroupResInfo->index += 1;

×

990

    for (int32_t j = 0; j < numOfExprs; ++j) {

×

991

      int32_t slotId = pExprInfo[j].base.resSchema.slotId;

×

993

      pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset);

×

995

      if (pCtx[j].fpSet.finalize) {

×

996

        int32_t tmpRes = pCtx[j].fpSet.finalize(&pCtx[j], pBlock);

×

997

        if (TAOS_FAILED(tmpRes)) {

×

998

          qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(tmpRes));

×

999

          QUERY_CHECK_CODE(code, lino, _end);

×

1001

      } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) {

×

1006

        SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId);

×

1007

        char*            in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo);

×

1008

        for (int32_t k = 0; k < pRow->numOfRows; ++k) {

×

1009

          code = colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes);

×

1010

          QUERY_CHECK_CODE(code, lino, _end);

×

1015

    if (pSessionKeys) {

×

1016

      SSessionKey key = {.groupId = groupId, .win = pRow->win};

×

1017

      for (int32_t j = 0; j < pRow->numOfRows; ++j) {

×

1018

        const void* px = taosArrayPush(pSessionKeys, &key);

×

1019

        QUERY_CHECK_NULL(px, code, lino, _end, terrno);

×

1023

    pBlock->info.rows += pRow->numOfRows;

×

1026

  pBlock->info.dataLoad = 1;

×

1027

  code = blockDataUpdateTsWindow(pBlock, 0);

×

1028

  QUERY_CHECK_CODE(code, lino, _end);

×

1030

_end:

×

1031

  if (code != TSDB_CODE_SUCCESS) {

×

1032

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

1034

×

1036

void doBuildStreamIntervalResult(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock,

×

1038

  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

×

1040

  pBlock->info.version = pTaskInfo->version;

×

1042

  blockDataCleanup(pBlock);

×

1043

  taosArrayClear(pSessionKeys);

×

1044

  if (!hasRemainResults(pGroupResInfo)) {

×

1045

    return;

×

1049

  pBlock->info.id.groupId = 0;

×

1050

  buildDataBlockFromGroupRes(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo, pSessionKeys);

×

1053

static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,

×

1055

  int32_t startPos = prevPosition + 1;

×

1056

  if (startPos == pDataBlockInfo->rows) {

×

1057

    startPos = -1;

×

1059

    *pNext = getFinalTimeWindow(primaryKeys[startPos], pInterval);

×

1061

  return startPos;

×

1064

static int32_t doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, uint64_t groupId,

×

1066

  int32_t                      code = TSDB_CODE_SUCCESS;

×

1067

  int32_t                      lino = 0;

×

1068

  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info;

×

1069

  pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);

×

1071

  SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo);

×

1072

  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;

×

1073

  SExprSupp*      pSup = &pOperator->exprSupp;

×

1074

  int32_t         numOfOutput = pSup->numOfExprs;

×

1075

  int32_t         step = 1;

×

1076

  TSKEY*          tsCols = NULL;

×

1077

  SRowBuffPos*    pResPos = NULL;

×

1078

  SResultRow*     pResult = NULL;

×

1079

  int32_t         forwardRows = 0;

×

1080

  int32_t         endRowId = pSDataBlock->info.rows - 1;

×

1082

  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);

×

1083

  tsCols = (int64_t*)pColDataInfo->pData;

×

1085

  void*            pPkVal = NULL;

×

1086

  int32_t          pkLen = 0;

×

1087

  SColumnInfoData* pPkColDataInfo = NULL;

×

1088

  if (hasSrcPrimaryKeyCol(&pInfo->basic)) {

×

1089

    pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex);

×

1092

  if (pSDataBlock->info.window.skey != tsCols[0] || pSDataBlock->info.window.ekey != tsCols[endRowId]) {

×

1093

    qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 ",maxKey %" PRId64,

×

1095

    code = blockDataUpdateTsWindow(pSDataBlock, pInfo->primaryTsIndex);

×

1096

    QUERY_CHECK_CODE(code, lino, _end);

×

1099

    if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) {

×

1100

      qError("table uid %" PRIu64 " data block timestamp is out of range! minKey %" PRId64 ",maxKey %" PRId64,

×

1105

  int32_t     startPos = 0;

×

1106

  TSKEY       ts = getStartTsKey(&pSDataBlock->info.window, tsCols);

×

1107

  STimeWindow nextWin = {0};

×

1108

  if (IS_FINAL_INTERVAL_OP(pOperator)) {

×

1109

    nextWin = getFinalTimeWindow(ts, &pInfo->interval);

×

1111

    nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC);

×

1113

  while (1) {

×

1114

    bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);

×

1115

    if (hasSrcPrimaryKeyCol(&pInfo->basic) && !IS_FINAL_INTERVAL_OP(pOperator) && pInfo->ignoreExpiredData &&

×

1116

        pSDataBlock->info.type != STREAM_PULL_DATA) {

×

1117

      pPkVal = colDataGetData(pPkColDataInfo, startPos);

×

1118

      pkLen = colDataGetRowLength(pPkColDataInfo, startPos);

×

1121

    if ((!IS_FINAL_INTERVAL_OP(pOperator) && pInfo->ignoreExpiredData && pSDataBlock->info.type != STREAM_PULL_DATA &&

×

1122

         checkExpiredData(&pInfo->stateStore, pInfo->pUpdateInfo, &pInfo->twAggSup, pSDataBlock->info.id.uid,

×

1123

                          nextWin.ekey, pPkVal, pkLen)) ||

×

1124

        !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) {

×

1125

      startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);

×

1126

      if (startPos < 0) {

×

1127

        break;

×

1129

      qDebug("===stream===ignore expired data, window end ts:%" PRId64 ", maxts - wartermak:%" PRId64, nextWin.ekey,

×

1131

      continue;

×

1134

    if (IS_FINAL_INTERVAL_OP(pOperator) && pInfo->numOfChild > 0) {

×

1135

      bool    ignore = true;

×

1136

      SWinKey winRes = {

×

1137

          .ts = nextWin.skey,

×

1140

      void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey));

×

1141

      if (isDeletedStreamWindow(&nextWin, groupId, pInfo->pState, &pInfo->twAggSup, &pInfo->stateStore) && isClosed &&

×

1142

          !chIds) {

×

1143

        SPullWindowInfo pull = {

×

1144

            .window = nextWin, .groupId = groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey};

×

1146

        if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) {

×

1147

          code = addPullWindow(pInfo->pPullDataMap, &winRes, pInfo->numOfChild);

×

1148

          QUERY_CHECK_CODE(code, lino, _end);

×

1150

          if (pInfo->destHasPrimaryKey) {

×

1151

            code = tSimpleHashPut(pInfo->pDeletedMap, &winRes, sizeof(SWinKey), NULL, 0);

×

1152

            QUERY_CHECK_CODE(code, lino, _end);

×

1156

        int32_t index = -1;

×

1157

        SArray* chArray = NULL;

×

1158

        int32_t chId = 0;

×

1159

        if (chIds) {

×

1160

          chArray = *(void**)chIds;

×

1161

          chId = getChildIndex(pSDataBlock);

×

1162

          index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);

×

1164

        if (index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA) {

×

1165

          ignore = false;

×

1169

      if (ignore) {

×

1170

        startPos = getNextQualifiedFinalWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, startPos);

×

1171

        if (startPos < 0) {

×

1172

          break;

×

1174

        continue;

×

1178

    int32_t winCode = TSDB_CODE_SUCCESS;

×

1179

    code = setIntervalOutputBuf(pInfo->pState, &nextWin, &pResPos, groupId, pSup->pCtx, numOfOutput,

×

1181

    QUERY_CHECK_CODE(code, lino, _end);

×

1183

    if (winCode != TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator) &&

×

1184

        BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {

×

1185

      SSessionKey key = {.win = nextWin, .groupId = groupId};

×

1186

      code = addIntervalAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &key, &pInfo->basic.notifyEventSup,

×

1188

      QUERY_CHECK_CODE(code, lino, _end);

×

1191

    pResult = (SResultRow*)pResPos->pRowBuff;

×

1193

    if (IS_FINAL_INTERVAL_OP(pOperator)) {

×

1194

      forwardRows = 1;

×

1196

      forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey,

×

1200

    SWinKey key = {

×

1201

        .ts = pResult->win.skey,

×

1205

    if (pInfo->destHasPrimaryKey && winCode == TSDB_CODE_SUCCESS && IS_NORMAL_INTERVAL_OP(pOperator)) {

×

1206

      code = tSimpleHashPut(pDeletedMap, &key, sizeof(SWinKey), NULL, 0);

×

1207

      QUERY_CHECK_CODE(code, lino, _end);

×

1210

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) {

×

1211

      code = saveWinResult(&key, pResPos, pUpdatedMap);

×

1212

      QUERY_CHECK_CODE(code, lino, _end);

×

1215

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

1216

      pResPos->beUpdated = true;

×

1217

      code = tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pResPos, POINTER_BYTES);

×

1218

      QUERY_CHECK_CODE(code, lino, _end);

×

1221

    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, 1);

×

1222

    code = applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos,

×

1223

                                           forwardRows, pSDataBlock->info.rows, numOfOutput);

×

1224

    QUERY_CHECK_CODE(code, lino, _end);

×

1225

    key.ts = nextWin.skey;

×

1227

    if (pInfo->delKey.ts > key.ts) {

×

1228

      pInfo->delKey = key;

×

1230

    int32_t prevEndPos = (forwardRows - 1) * step + startPos;

×

1231

    if (IS_FINAL_INTERVAL_OP(pOperator)) {

×

1232

      startPos = getNextQualifiedFinalWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos);

×

1235

          getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC);

×

1237

    if (startPos < 0) {

×

1238

      break;

×

1241

_end:

×

1242

  if (code != TSDB_CODE_SUCCESS) {

×

1243

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

1245

  return code;

×

1248

int winPosCmprImpl(const void* pKey1, const void* pKey2) {

×

1249

  SRowBuffPos* pos1 = *(SRowBuffPos**)pKey1;

×

1250

  SRowBuffPos* pos2 = *(SRowBuffPos**)pKey2;

×

1251

  SWinKey*     pWin1 = (SWinKey*)pos1->pKey;

×

1252

  SWinKey*     pWin2 = (SWinKey*)pos2->pKey;

×

1254

  if (pWin1->groupId > pWin2->groupId) {

×

1255

    return 1;

×

1256

  } else if (pWin1->groupId < pWin2->groupId) {

×

1257

    return -1;

×

1260

  if (pWin1->ts > pWin2->ts) {

×

1261

    return 1;

×

1262

  } else if (pWin1->ts < pWin2->ts) {

×

1263

    return -1;

×

1266

  return 0;

×

1269

static void resetUnCloseWinInfo(SSHashObj* winMap) {

×

1270

  void*   pIte = NULL;

×

1271

  int32_t iter = 0;

×

1272

  while ((pIte = tSimpleHashIterate(winMap, pIte, &iter)) != NULL) {

×

1273

    SRowBuffPos* pPos = *(SRowBuffPos**)pIte;

×

1274

    pPos->beUsed = true;

×

1276

×

1278

int32_t encodeSWinKey(void** buf, SWinKey* key) {

×

1279

  int32_t tlen = 0;

×

1280

  tlen += taosEncodeFixedI64(buf, key->ts);

×

1281

  tlen += taosEncodeFixedU64(buf, key->groupId);

×

1282

  return tlen;

×

1285

void* decodeSWinKey(void* buf, SWinKey* key) {

×

1286

  buf = taosDecodeFixedI64(buf, &key->ts);

×

1287

  buf = taosDecodeFixedU64(buf, &key->groupId);

×

1288

  return buf;

×

1291

int32_t encodeSTimeWindowAggSupp(void** buf, STimeWindowAggSupp* pTwAggSup) {

×

1292

  int32_t tlen = 0;

×

1293

  tlen += taosEncodeFixedI64(buf, pTwAggSup->minTs);

×

1294

  tlen += taosEncodeFixedI64(buf, pTwAggSup->maxTs);

×

1295

  return tlen;

×

1298

void* decodeSTimeWindowAggSupp(void* buf, STimeWindowAggSupp* pTwAggSup) {

×

1299

  buf = taosDecodeFixedI64(buf, &pTwAggSup->minTs);

×

1300

  buf = taosDecodeFixedI64(buf, &pTwAggSup->maxTs);

×

1301

  return buf;

×

1304

int32_t encodeSTimeWindow(void** buf, STimeWindow* pWin) {

×

1305

  int32_t tlen = 0;

×

1306

  tlen += taosEncodeFixedI64(buf, pWin->skey);

×

1307

  tlen += taosEncodeFixedI64(buf, pWin->ekey);

×

1308

  return tlen;

×

1311

void* decodeSTimeWindow(void* buf, STimeWindow* pWin) {

×

1312

  buf = taosDecodeFixedI64(buf, &pWin->skey);

×

1313

  buf = taosDecodeFixedI64(buf, &pWin->ekey);

×

1314

  return buf;

×

1317

int32_t encodeSPullWindowInfo(void** buf, SPullWindowInfo* pPullInfo) {

×

1318

  int32_t tlen = 0;

×

1319

  tlen += encodeSTimeWindow(buf, &pPullInfo->calWin);

×

1320

  tlen += taosEncodeFixedU64(buf, pPullInfo->groupId);

×

1321

  tlen += encodeSTimeWindow(buf, &pPullInfo->window);

×

1322

  return tlen;

×

1325

void* decodeSPullWindowInfo(void* buf, SPullWindowInfo* pPullInfo) {

×

1326

  buf = decodeSTimeWindow(buf, &pPullInfo->calWin);

×

1327

  buf = taosDecodeFixedU64(buf, &pPullInfo->groupId);

×

1328

  buf = decodeSTimeWindow(buf, &pPullInfo->window);

×

1329

  return buf;

×

1332

int32_t encodeSPullWindowInfoArray(void** buf, SArray* pPullInfos) {

×

1333

  int32_t tlen = 0;

×

1334

  int32_t size = taosArrayGetSize(pPullInfos);

×

1335

  tlen += taosEncodeFixedI32(buf, size);

×

1336

  for (int32_t i = 0; i < size; i++) {

×

1337

    void* pItem = taosArrayGet(pPullInfos, i);

×

1338

    tlen += encodeSPullWindowInfo(buf, pItem);

×

1340

  return tlen;

×

1343

int32_t decodeSPullWindowInfoArray(void* buf, SArray* pPullInfos, void** ppBuf) {

×

1344

  int32_t code = TSDB_CODE_SUCCESS;

×

1345

  int32_t lino = 0;

×

1346

  int32_t size = 0;

×

1347

  buf = taosDecodeFixedI32(buf, &size);

×

1348

  for (int32_t i = 0; i < size; i++) {

×

1349

    SPullWindowInfo item = {0};

×

1350

    buf = decodeSPullWindowInfo(buf, &item);

×

1351

    void* tmp = taosArrayPush(pPullInfos, &item);

×

1352

    if (!tmp) {

×

1353

      code = terrno;

×

1354

      QUERY_CHECK_CODE(code, lino, _end);

×

1357

  (*ppBuf) = buf;

×

1359

_end:

×

1360

  if (code != TSDB_CODE_SUCCESS) {

×

1361

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

1363

  return code;

×

1366

int32_t doStreamIntervalEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOperator) {

×

1367

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1368

  if (!pInfo) {

×

1369

    return 0;

×

1372

  void* pData = (buf == NULL) ? NULL : *buf;

×

1375

  int32_t tlen = 0;

×

1376

  int32_t mapSize = tSimpleHashGetSize(pInfo->aggSup.pResultRowHashTable);

×

1377

  tlen += taosEncodeFixedI32(buf, mapSize);

×

1378

  void*   pIte = NULL;

×

1379

  size_t  keyLen = 0;

×

1380

  int32_t iter = 0;

×

1381

  while ((pIte = tSimpleHashIterate(pInfo->aggSup.pResultRowHashTable, pIte, &iter)) != NULL) {

×

1382

    void* key = tSimpleHashGetKey(pIte, &keyLen);

×

1383

    tlen += encodeSWinKey(buf, key);

×

1387

  tlen += encodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);

×

1390

  int32_t size = taosHashGetSize(pInfo->pPullDataMap);

×

1391

  tlen += taosEncodeFixedI32(buf, size);

×

1392

  pIte = NULL;

×

1393

  keyLen = 0;

×

1394

  while ((pIte = taosHashIterate(pInfo->pPullDataMap, pIte)) != NULL) {

×

1395

    void* key = taosHashGetKey(pIte, &keyLen);

×

1396

    tlen += encodeSWinKey(buf, key);

×

1397

    SArray* pArray = *(SArray**)pIte;

×

1398

    int32_t chSize = taosArrayGetSize(pArray);

×

1399

    tlen += taosEncodeFixedI32(buf, chSize);

×

1400

    for (int32_t i = 0; i < chSize; i++) {

×

1401

      void* pChItem = taosArrayGet(pArray, i);

×

1402

      tlen += taosEncodeFixedI32(buf, *(int32_t*)pChItem);

×

1407

  tlen += encodeSPullWindowInfoArray(buf, pInfo->pPullWins);

×

1410

  tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);

×

1413

  tlen += encodeStreamBasicInfo(buf, &pInfo->basic);

×

1416

  if (buf) {

×

1417

    uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));

×

1418

    tlen += taosEncodeFixedU32(buf, cksum);

×

1420

    tlen += sizeof(uint32_t);

×

1423

  return tlen;

×

1426

void doStreamIntervalDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator) {

×

1427

  int32_t                      code = TSDB_CODE_SUCCESS;

×

1428

  int32_t                      lino = 0;

×

1429

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1430

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

1431

  void*                        pDataEnd = POINTER_SHIFT(buf, len);

×

1432

  if (!pInfo) {

×

1433

    code = TSDB_CODE_FAILED;

×

1434

    QUERY_CHECK_CODE(code, lino, _end);

×

1438

  int32_t dataLen = len - sizeof(uint32_t);

×

1439

  void*   pCksum = POINTER_SHIFT(buf, dataLen);

×

1440

  if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) {

×

1441

    code = TSDB_CODE_FAILED;

×

1442

    QUERY_CHECK_CODE(code, lino, _end);

×

1444

  pDataEnd = pCksum;

×

1447

  int32_t mapSize = 0;

×

1448

  buf = taosDecodeFixedI32(buf, &mapSize);

×

1449

  for (int32_t i = 0; i < mapSize; i++) {

×

1450

    SWinKey key = {0};

×

1451

    buf = decodeSWinKey(buf, &key);

×

1452

    SRowBuffPos* pPos = NULL;

×

1453

    int32_t      resSize = pInfo->aggSup.resultRowSize;

×

1454

    int32_t      winCode = TSDB_CODE_SUCCESS;

×

1455

    code = pInfo->stateStore.streamStateAddIfNotExist(pInfo->pState, &key, (void**)&pPos, &resSize, &winCode);

×

1456

    QUERY_CHECK_CODE(code, lino, _end);

×

1457

    QUERY_CHECK_CONDITION((winCode == TSDB_CODE_SUCCESS), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);

×

1459

    code = tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pPos, POINTER_BYTES);

×

1460

    QUERY_CHECK_CODE(code, lino, _end);

×

1464

  buf = decodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);

×

1467

  int32_t size = 0;

×

1468

  buf = taosDecodeFixedI32(buf, &size);

×

1469

  for (int32_t i = 0; i < size; i++) {

×

1470

    SWinKey key = {0};

×

1471

    SArray* pArray = taosArrayInit(0, sizeof(int32_t));

×

1472

    if (!pArray) {

×

1473

      code = terrno;

×

1474

      QUERY_CHECK_CODE(code, lino, _end);

×

1477

    buf = decodeSWinKey(buf, &key);

×

1478

    int32_t chSize = 0;

×

1479

    buf = taosDecodeFixedI32(buf, &chSize);

×

1480

    for (int32_t i = 0; i < chSize; i++) {

×

1481

      int32_t chId = 0;

×

1482

      buf = taosDecodeFixedI32(buf, &chId);

×

1483

      void* tmp = taosArrayPush(pArray, &chId);

×

1484

      if (!tmp) {

×

1485

        code = terrno;

×

1486

        QUERY_CHECK_CODE(code, lino, _end);

×

1489

    code = taosHashPut(pInfo->pPullDataMap, &key, sizeof(SWinKey), &pArray, POINTER_BYTES);

×

1490

    QUERY_CHECK_CODE(code, lino, _end);

×

1494

  code = decodeSPullWindowInfoArray(buf, pInfo->pPullWins, &buf);

×

1495

  QUERY_CHECK_CODE(code, lino, _end);

×

1498

  buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);

×

1501

  if (buf < pDataEnd) {

×

1502

    code = decodeStreamBasicInfo(&buf, &pInfo->basic);

×

1503

    QUERY_CHECK_CODE(code, lino, _end);

×

1506

_end:

×

1507

  if (code != TSDB_CODE_SUCCESS) {

×

1508

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

1510

×

1512

void doStreamIntervalSaveCheckpoint(SOperatorInfo* pOperator) {

×

1513

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1514

  if (needSaveStreamOperatorInfo(&pInfo->basic)) {

×

1515

    int32_t len = doStreamIntervalEncodeOpState(NULL, 0, pOperator);

×

1516

    void*   buf = taosMemoryCalloc(1, len);

×

1517

    if (!buf) {

×

1518

      qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno));

×

1519

      return;

×

1521

    void* pBuf = buf;

×

1522

    len = doStreamIntervalEncodeOpState(&pBuf, len, pOperator);

×

1523

    pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_INTERVAL_OP_CHECKPOINT_NAME,

×

1525

    taosMemoryFree(buf);

×

1526

    saveStreamOperatorStateComplete(&pInfo->basic);

×

1530

int32_t copyIntervalDeleteKey(SSHashObj* pMap, SArray* pWins) {

×

1531

  int32_t code = TSDB_CODE_SUCCESS;

×

1532

  int32_t lino = 0;

×

1533

  void*   pIte = NULL;

×

1534

  int32_t iter = 0;

×

1535

  while ((pIte = tSimpleHashIterate(pMap, pIte, &iter)) != NULL) {

×

1536

    void* pKey = tSimpleHashGetKey(pIte, NULL);

×

1537

    void* tmp = taosArrayPush(pWins, pKey);

×

1538

    if (!tmp) {

×

1539

      code = terrno;

×

1540

      QUERY_CHECK_CODE(code, lino, _end);

×

1543

  tSimpleHashClear(pMap);

×

1545

_end:

×

1546

  if (code != TSDB_CODE_SUCCESS) {

×

1547

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

1549

  return code;

×

1552

static int32_t buildIntervalResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

1553

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1554

  int32_t                      code = TSDB_CODE_SUCCESS;

×

1555

  int32_t                      lino = 0;

×

1556

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

1557

  uint16_t                     opType = pOperator->operatorType;

×

1558

  SStreamNotifyEventSupp*      pNotifySup = &pInfo->basic.notifyEventSup;

×

1559

  STaskNotifyEventStat*        pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;

×

1560

  bool                         addNotifyEvent = false;

×

1563

  if (isTaskKilled(pTaskInfo)) {

×

1564

    (*ppRes) = NULL;

×

1565

    return code;

×

1568

  addNotifyEvent = IS_NORMAL_INTERVAL_OP(pOperator) &&

×

1569

                   BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);

×

1570

  if (IS_FINAL_INTERVAL_OP(pOperator)) {

×

1571

    doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes);

×

1572

    if (pInfo->pPullDataRes->info.rows != 0) {

×

1574

      printDataBlock(pInfo->pPullDataRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));

×

1575

      (*ppRes) = pInfo->pPullDataRes;

×

1576

      return code;

×

1580

  doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);

×

1581

  if (pInfo->pDelRes->info.rows != 0) {

×

1583

    printDataBlock(pInfo->pDelRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));

×

1584

    if (addNotifyEvent) {

×

1585

      code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);

×

1586

      QUERY_CHECK_CODE(code, lino, _end);

×

1588

    (*ppRes) = pInfo->pDelRes;

×

1589

    return code;

×

1592

  doBuildStreamIntervalResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo,

×

1594

  if (pInfo->binfo.pRes->info.rows != 0) {

×

1595

    printDataBlock(pInfo->binfo.pRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));

×

1596

    if (addNotifyEvent) {

×

1597

      code = addAggResultNotifyEvent(pInfo->binfo.pRes, pNotifySup->pSessionKeys,

×

1598

                                     pTaskInfo->streamInfo.notifyResultSchema, pNotifySup, pNotifyEventStat);

×

1599

      QUERY_CHECK_CODE(code, lino, _end);

×

1601

    (*ppRes) = pInfo->binfo.pRes;

×

1602

    return code;

×

1605

  code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);

×

1606

  QUERY_CHECK_CODE(code, lino, _end);

×

1607

  if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {

×

1608

    printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

1609

    (*ppRes) = pNotifySup->pEventBlock;

×

1610

    return code;

×

1613

  code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);

×

1614

  QUERY_CHECK_CODE(code, lino, _end);

×

1616

_end:

×

1617

  if (code != TSDB_CODE_SUCCESS) {

×

1618

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

1620

  (*ppRes) = NULL;

×

1621

  return code;

×

1624

int32_t copyUpdateResult(SSHashObj** ppWinUpdated, SArray* pUpdated, __compar_fn_t compar) {

×

1625

  int32_t code = TSDB_CODE_SUCCESS;

×

1626

  int32_t lino = 0;

×

1627

  void*   pIte = NULL;

×

1628

  int32_t iter = 0;

×

1629

  while ((pIte = tSimpleHashIterate(*ppWinUpdated, pIte, &iter)) != NULL) {

×

1630

    void* tmp = taosArrayPush(pUpdated, pIte);

×

1631

    if (!tmp) {

×

1632

      code = terrno;

×

1633

      QUERY_CHECK_CODE(code, lino, _end);

×

1636

  taosArraySort(pUpdated, compar);

×

1637

  tSimpleHashCleanup(*ppWinUpdated);

×

1638

  *ppWinUpdated = NULL;

×

1640

_end:

×

1641

  if (code != TSDB_CODE_SUCCESS) {

×

1642

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

1644

  return code;

×

1647

static int32_t doStreamFinalIntervalAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

1648

  int32_t                      code = TSDB_CODE_SUCCESS;

×

1649

  int32_t                      lino = 0;

×

1650

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1651

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

1652

  SStorageAPI*                 pAPI = &pOperator->pTaskInfo->storageAPI;

×

1654

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

1655

  SExprSupp*     pSup = &pOperator->exprSupp;

×

1657

  qDebug("stask:%s  %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);

×

1659

  if (pOperator->status == OP_EXEC_DONE) {

×

1660

    (*ppRes) = NULL;

×

1661

    return code;

×

1662

  } else if (pOperator->status == OP_RES_TO_RETURN) {

×

1663

    SSDataBlock* resBlock = NULL;

×

1664

    code = buildIntervalResult(pOperator, &resBlock);

×

1665

    QUERY_CHECK_CODE(code, lino, _end);

×

1666

    if (resBlock != NULL) {

×

1667

      (*ppRes) = resBlock;

×

1668

      return code;

×

1671

    if (pInfo->recvGetAll) {

×

1672

      pInfo->recvGetAll = false;

×

1673

      resetUnCloseWinInfo(pInfo->aggSup.pResultRowHashTable);

×

1676

    if (pInfo->reCkBlock) {

×

1677

      pInfo->reCkBlock = false;

×

1678

      printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

1679

      (*ppRes) = pInfo->pCheckpointRes;

×

1680

      return code;

×

1683

    setStreamOperatorCompleted(pOperator);

×

1684

    if (!IS_FINAL_INTERVAL_OP(pOperator)) {

×

1685

      clearFunctionContext(&pOperator->exprSupp);

×

1687

      clearStreamIntervalOperator(pInfo);

×

1688

      qDebug("===stream===clear semi operator");

×

1690

    (*ppRes) = NULL;

×

1691

    return code;

×

1693

    if (!IS_FINAL_INTERVAL_OP(pOperator)) {

×

1694

      SSDataBlock* resBlock = NULL;

×

1695

      code = buildIntervalResult(pOperator, &resBlock);

×

1696

      QUERY_CHECK_CODE(code, lino, _end);

×

1697

      if (resBlock != NULL) {

×

1698

        (*ppRes) = resBlock;

×

1699

        return code;

×

1702

      if (pInfo->recvRetrive) {

×

1703

        pInfo->recvRetrive = false;

×

1704

        printDataBlock(pInfo->pMidRetriveRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

1705

        (*ppRes) = pInfo->pMidRetriveRes;

×

1706

        return code;

×

1711

  if (!pInfo->pUpdated) {

×

1712

    pInfo->pUpdated = taosArrayInit(4096, POINTER_BYTES);

×

1713

    QUERY_CHECK_NULL(pInfo->pUpdated, code, lino, _end, terrno);

×

1715

  if (!pInfo->pUpdatedMap) {

×

1716

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

1717

    pInfo->pUpdatedMap = tSimpleHashInit(4096, hashFn);

×

1718

    QUERY_CHECK_NULL(pInfo->pUpdatedMap, code, lino, _end, terrno);

×

1721

  while (1) {

×

1722

    if (isTaskKilled(pTaskInfo)) {

×

1723

      qInfo("===stream=== %s task is killed, code %s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code));

×

1724

      (*ppRes) = NULL;

×

1725

      return code;

×

1728

    SSDataBlock* pBlock = NULL;

×

1729

    code = downstream->fpSet.getNextFn(downstream, &pBlock);

×

1730

    QUERY_CHECK_CODE(code, lino, _end);

×

1732

    if (pBlock == NULL) {

×

1733

      pOperator->status = OP_RES_TO_RETURN;

×

1734

      qDebug("===stream===return data:%s. recv datablock num:%" PRIu64, getStreamOpName(pOperator->operatorType),

×

1736

      pInfo->numOfDatapack = 0;

×

1737

      break;

×

1740

    pInfo->numOfDatapack++;

×

1741

    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));

×

1742

    setStreamOperatorState(&pInfo->basic, pBlock->info.type);

×

1744

    if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) {

×

1745

      pInfo->binfo.pRes->info.type = pBlock->info.type;

×

1746

    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||

×

1747

               pBlock->info.type == STREAM_CLEAR) {

×

1748

      SArray* delWins = taosArrayInit(8, sizeof(SWinKey));

×

1749

      QUERY_CHECK_NULL(delWins, code, lino, _end, terrno);

×

1750

      SHashObj* finalMap = IS_FINAL_INTERVAL_OP(pOperator) ? pInfo->pFinalPullDataMap : NULL;

×

1751

      code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, finalMap);

×

1752

      QUERY_CHECK_CODE(code, lino, _end);

×

1754

      if (IS_FINAL_INTERVAL_OP(pOperator)) {

×

1755

        int32_t chId = getChildIndex(pBlock);

×

1756

        code = addRetriveWindow(delWins, pInfo, chId);

×

1757

        QUERY_CHECK_CODE(code, lino, _end);

×

1759

        if (pBlock->info.type != STREAM_CLEAR) {

×

1760

          void* tmp = taosArrayAddAll(pInfo->pDelWins, delWins);

×

1761

          if (!tmp && taosArrayGetSize(delWins) > 0) {

×

1762

            code = TSDB_CODE_OUT_OF_MEMORY;

×

1763

            QUERY_CHECK_CODE(code, lino, _end);

×

1766

        taosArrayDestroy(delWins);

×

1767

        continue;

×

1769

      removeResults(delWins, pInfo->pUpdatedMap);

×

1770

      void* tmp = taosArrayAddAll(pInfo->pDelWins, delWins);

×

1771

      if (!tmp && taosArrayGetSize(delWins) > 0) {

×

1772

        code = TSDB_CODE_OUT_OF_MEMORY;

×

1773

        QUERY_CHECK_CODE(code, lino, _end);

×

1775

      taosArrayDestroy(delWins);

×

1777

      doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);

×

1778

      if (pInfo->pDelRes->info.rows != 0) {

×

1780

        printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

1781

        if (pBlock->info.type == STREAM_CLEAR) {

×

1782

          pInfo->pDelRes->info.type = STREAM_CLEAR;

×

1784

          pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;

×

1786

        (*ppRes) = pInfo->pDelRes;

×

1787

        return code;

×

1790

      break;

×

1791

    } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_INTERVAL_OP(pOperator)) {

×

1792

      pInfo->recvGetAll = true;

×

1793

      code = getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap);

×

1794

      QUERY_CHECK_CODE(code, lino, _end);

×

1795

      continue;

×

1796

    } else if (pBlock->info.type == STREAM_RETRIEVE) {

×

1797

      if (!IS_FINAL_INTERVAL_OP(pOperator)) {

×

1798

        pInfo->recvRetrive = true;

×

1799

        code = copyDataBlock(pInfo->pMidRetriveRes, pBlock);

×

1800

        QUERY_CHECK_CODE(code, lino, _end);

×

1802

        pInfo->pMidRetriveRes->info.type = STREAM_MID_RETRIEVE;

×

1803

        code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, NULL, pInfo->pUpdatedMap, NULL);

×

1804

        QUERY_CHECK_CODE(code, lino, _end);

×

1805

        break;

×

1807

      continue;

×

1808

    } else if (pBlock->info.type == STREAM_PULL_OVER && IS_FINAL_INTERVAL_OP(pOperator)) {

×

1809

      code = processPullOver(pBlock, pInfo->pPullDataMap, pInfo->pFinalPullDataMap, &pInfo->interval, pInfo->pPullWins,

×

1811

      QUERY_CHECK_CODE(code, lino, _end);

×

1812

      continue;

×

1813

    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {

×

1814

      (*ppRes) = pBlock;

×

1815

      return code;

×

1816

    } else if (pBlock->info.type == STREAM_CHECKPOINT) {

×

1817

      pAPI->stateStore.streamStateCommit(pInfo->pState);

×

1818

      doStreamIntervalSaveCheckpoint(pOperator);

×

1819

      code = copyDataBlock(pInfo->pCheckpointRes, pBlock);

×

1820

      QUERY_CHECK_CODE(code, lino, _end);

×

1822

      continue;

×

1823

    } else if (IS_FINAL_INTERVAL_OP(pOperator) && pBlock->info.type == STREAM_MID_RETRIEVE) {

×

1824

      continue;

×

1826

      if (pBlock->info.type != STREAM_INVALID) {

×

1827

        code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;

×

1828

        QUERY_CHECK_CODE(code, lino, _end);

×

1832

    if (pInfo->scalarSupp.pExprInfo != NULL) {

×

1833

      SExprSupp* pExprSup = &pInfo->scalarSupp;

×

1834

      code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);

×

1835

      QUERY_CHECK_CODE(code, lino, _end);

×

1837

    code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

1838

    QUERY_CHECK_CODE(code, lino, _end);

×

1840

    code = doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pInfo->pUpdatedMap, pInfo->pDeletedMap);

×

1841

    if (code == TSDB_CODE_STREAM_INTERNAL_ERROR) {

×

1842

      code = TSDB_CODE_SUCCESS;

×

1843

      pOperator->status = OP_RES_TO_RETURN;

×

1844

      break;

×

1846

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);

×

1847

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.watermark);

×

1848

    pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey);

×

1851

  if (IS_FINAL_INTERVAL_OP(pOperator) && !pInfo->destHasPrimaryKey) {

×

1852

    removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins);

×

1854

  if (IS_FINAL_INTERVAL_OP(pOperator)) {

×

1855

    code = closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval,

×

1857

    QUERY_CHECK_CODE(code, lino, _end);

×

1859

    if (pInfo->destHasPrimaryKey) {

×

1860

      code = copyIntervalDeleteKey(pInfo->pDeletedMap, pInfo->pDelWins);

×

1861

      QUERY_CHECK_CODE(code, lino, _end);

×

1864

  pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs;

×

1866

  code = copyUpdateResult(&pInfo->pUpdatedMap, pInfo->pUpdated, winPosCmprImpl);

×

1867

  QUERY_CHECK_CODE(code, lino, _end);

×

1869

  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);

×

1870

  pInfo->pUpdated = NULL;

×

1871

  code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);

×

1872

  QUERY_CHECK_CODE(code, lino, _end);

×

1874

  SSDataBlock* resBlock = NULL;

×

1875

  code = buildIntervalResult(pOperator, &resBlock);

×

1876

  QUERY_CHECK_CODE(code, lino, _end);

×

1877

  if (resBlock != NULL) {

×

1878

    (*ppRes) = resBlock;

×

1879

    return code;

×

1882

  if (pInfo->recvRetrive) {

×

1883

    pInfo->recvRetrive = false;

×

1884

    printDataBlock(pInfo->pMidRetriveRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

1885

    (*ppRes) = pInfo->pMidRetriveRes;

×

1886

    return code;

×

1889

_end:

×

1890

  if (code != TSDB_CODE_SUCCESS) {

×

1891

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

1892

    pTaskInfo->code = code;

×

1893

    T_LONG_JMP(pTaskInfo->env, code);

×

1895

  setStreamOperatorCompleted(pOperator);

×

1896

  (*ppRes) = NULL;

×

1897

  return code;

×

1900

int64_t getDeleteMark(SWindowPhysiNode* pWinPhyNode, int64_t interval) {

×

1901

  if (pWinPhyNode->deleteMark <= 0) {

×

1902

    return DEAULT_DELETE_MARK;

×

1904

  int64_t deleteMark = TMAX(pWinPhyNode->deleteMark, pWinPhyNode->watermark);

×

1905

  deleteMark = TMAX(deleteMark, interval);

×

1906

  return deleteMark;

×

1909

int64_t getDeleteMarkFromOption(SStreamNodeOption* pOption) {

×

1910

  if (pOption->deleteMark <= 0) {

×

1911

    return DEAULT_DELETE_MARK;

×

1913

  int64_t deleteMark = TMAX(pOption->deleteMark, pOption->watermark);

×

1914

  return deleteMark;

×

1917

TSKEY compareTs(void* pKey) {

×

1918

  SWinKey* pWinKey = (SWinKey*)pKey;

×

1919

  return pWinKey->ts;

×

1922

static int32_t getSelectivityBufSize(SqlFunctionCtx* pCtx) {

×

1923

  if (pCtx->subsidiaries.rowLen == 0) {

×

1924

    int32_t rowLen = 0;

×

1925

    for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) {

×

1926

      SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j];

×

1927

      rowLen += pc->pExpr->base.resSchema.bytes;

×

1930

    return rowLen + pCtx->subsidiaries.num * sizeof(bool);

×

1932

    return pCtx->subsidiaries.rowLen;

×

1936

static int32_t getMaxFunResSize(SExprSupp* pSup, int32_t numOfCols) {

×

1937

  int32_t size = 0;

×

1938

  for (int32_t i = 0; i < numOfCols; ++i) {

×

1939

    int32_t resSize = getSelectivityBufSize(pSup->pCtx + i);

×

1940

    size = TMAX(size, resSize);

×

1942

  return size;

×

1945

static void streamIntervalReleaseState(SOperatorInfo* pOperator) {

×

1946

  if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL &&

×

1947

      pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {

×

1948

    SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1949

    int32_t                      resSize = sizeof(TSKEY);

×

1950

    pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME,

×

1951

                                          strlen(STREAM_INTERVAL_OP_STATE_NAME), &pInfo->twAggSup.maxTs, resSize);

×

1953

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1954

  SStorageAPI*                 pAPI = &pOperator->pTaskInfo->storageAPI;

×

1955

  pAPI->stateStore.streamStateCommit(pInfo->pState);

×

1956

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

1957

  if (downstream->fpSet.releaseStreamStateFn) {

×

1958

    downstream->fpSet.releaseStreamStateFn(downstream);

×

1960

×

1962

void streamIntervalReloadState(SOperatorInfo* pOperator) {

×

1963

  int32_t                      code = TSDB_CODE_SUCCESS;

×

1964

  int32_t                      lino = 0;

×

1965

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

1966

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

1967

  if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL &&

×

1968

      pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {

×

1969

    int32_t size = 0;

×

1970

    void*   pBuf = NULL;

×

1971

    code = pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_STATE_NAME,

×

1973

    QUERY_CHECK_CODE(code, lino, _end);

×

1975

    TSKEY ts = *(TSKEY*)pBuf;

×

1976

    taosMemoryFree(pBuf);

×

1977

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);

×

1978

    pInfo->stateStore.streamStateReloadInfo(pInfo->pState, ts);

×

1980

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

1981

  if (downstream->fpSet.reloadStreamStateFn) {

×

1982

    downstream->fpSet.reloadStreamStateFn(downstream);

×

1984

  reloadFromDownStream(downstream, pInfo);

×

1986

_end:

×

1987

  if (code != TSDB_CODE_SUCCESS) {

×

1988

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

1990

×

1992

int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,

×

1994

  QRY_PARAM_CHECK(pOptrInfo);

×

1996

  int32_t                      code = TSDB_CODE_SUCCESS;

×

1997

  int32_t                      lino = 0;

×

1998

  SIntervalPhysiNode*          pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;

×

1999

  SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo));

×

2000

  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));

×

2001

  if (pInfo == NULL || pOperator == NULL) {

×

2002

    code = terrno;

×

2003

    QUERY_CHECK_CODE(code, lino, _error);

×

2006

  pOperator->exprSupp.hasWindowOrGroup = true;

×

2007

  pOperator->pTaskInfo = pTaskInfo;

×

2008

  SStorageAPI* pAPI = &pTaskInfo->storageAPI;

×

2010

  pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval,

×

2011

                                .sliding = pIntervalPhyNode->sliding,

×

2012

                                .intervalUnit = pIntervalPhyNode->intervalUnit,

×

2013

                                .slidingUnit = pIntervalPhyNode->slidingUnit,

×

2014

                                .offset = pIntervalPhyNode->offset,

×

2015

                                .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision,

×

2016

                                .timeRange = pIntervalPhyNode->timeRange};

×

2017

  calcIntervalAutoOffset(&pInfo->interval);

×

2018

  pInfo->twAggSup = (STimeWindowAggSupp){

×

2019

      .waterMark = pIntervalPhyNode->window.watermark,

×

2020

      .calTrigger = pIntervalPhyNode->window.triggerType,

×

2023

      .deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval),

×

2027

  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;

×

2028

  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;

×

2029

  initResultSizeInfo(&pOperator->resultInfo, 4096);

×

2030

  if (pIntervalPhyNode->window.pExprs != NULL) {

×

2031

    int32_t    numOfScalar = 0;

×

2032

    SExprInfo* pScalarExprInfo = NULL;

×

2034

    code = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalar);

×

2035

    QUERY_CHECK_CODE(code, lino, _error);

×

2037

    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);

×

2038

    QUERY_CHECK_CODE(code, lino, _error);

×

2041

  SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);

×

2042

  QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);

×

2043

  initBasicInfo(&pInfo->binfo, pResBlock);

×

2045

  pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState));

×

2046

  QUERY_CHECK_NULL(pInfo->pState, code, lino, _error, terrno);

×

2047

  qInfo("open state %p", pInfo->pState);

×

2048

  pAPI->stateStore.streamStateCopyBackend(pTaskInfo->streamInfo.pState, pInfo->pState);

×

2051

  qInfo("copy state %p to %p", pTaskInfo->streamInfo.pState, pInfo->pState);

×

2053

  pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsIndex);

×

2055

  int32_t    numOfCols = 0;

×

2056

  SExprInfo* pExprInfo = NULL;

×

2057

  code = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);

×

2058

  QUERY_CHECK_CODE(code, lino, _error);

×

2060

  code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,

×

2061

                    pInfo->pState, &pTaskInfo->storageAPI.functionStore);

×

2062

  QUERY_CHECK_CODE(code, lino, _error);

×

2064

  code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

×

2065

  QUERY_CHECK_CODE(code, lino, _error);

×

2066

  initResultRowInfo(&pInfo->binfo.resultRowInfo);

×

2068

  pInfo->numOfChild = numOfChild;

×

2069

  pInfo->pPhyNode = NULL;

×

2070

  code = nodesCloneNode((SNode*)pPhyNode, (SNode**)&pInfo->pPhyNode);

×

2071

  if (TSDB_CODE_SUCCESS != code) {

×

2072

    goto _error;

×

2075

  pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo));

×

2076

  QUERY_CHECK_NULL(pInfo->pPullWins, code, lino, _error, terrno);

×

2077

  pInfo->pullIndex = 0;

×

2078

  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

2079

  pInfo->pPullDataMap = taosHashInit(64, hashFn, true, HASH_NO_LOCK);

×

2080

  pInfo->pFinalPullDataMap = taosHashInit(64, hashFn, true, HASH_NO_LOCK);

×

2082

  code = createSpecialDataBlock(STREAM_RETRIEVE, &pInfo->pPullDataRes);

×

2083

  QUERY_CHECK_CODE(code, lino, _error);

×

2085

  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;

×

2086

  pInfo->ignoreExpiredDataSaved = false;

×

2087

  code = createSpecialDataBlock(STREAM_DELETE_RESULT, &pInfo->pDelRes);

×

2088

  QUERY_CHECK_CODE(code, lino, _error);

×

2090

  pInfo->delIndex = 0;

×

2091

  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));

×

2092

  QUERY_CHECK_NULL(pInfo->pDelWins, code, lino, _error, terrno);

×

2093

  pInfo->delKey.ts = INT64_MAX;

×

2094

  pInfo->delKey.groupId = 0;

×

2095

  pInfo->numOfDatapack = 0;

×

2096

  pInfo->pUpdated = NULL;

×

2097

  pInfo->pUpdatedMap = NULL;

×

2098

  pInfo->stateStore = pTaskInfo->storageAPI.stateStore;

×

2099

  int32_t funResSize = getMaxFunResSize(&pOperator->exprSupp, numOfCols);

×

2100

  pInfo->pState->pFileState = NULL;

×

2103

  pInfo->pState->pResultRowStore.resultRowGet = getResultRowFromBuf;

×

2104

  pInfo->pState->pResultRowStore.resultRowPut = putResultRowToBuf;

×

2105

  pInfo->pState->pExprSupp = &pOperator->exprSupp;

×

2108

      pAPI->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize,

×

2109

                                           compareTs, pInfo->pState, pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo),

×

2110

                                           pHandle->checkpointId, STREAM_STATE_BUFF_HASH, &pInfo->pState->pFileState);

×

2111

  QUERY_CHECK_CODE(code, lino, _error);

×

2113

  pInfo->dataVersion = 0;

×

2114

  pInfo->recvGetAll = false;

×

2115

  pInfo->recvPullover = false;

×

2116

  pInfo->recvRetrive = false;

×

2118

  code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);

×

2119

  QUERY_CHECK_CODE(code, lino, _error);

×

2120

  code = createSpecialDataBlock(STREAM_MID_RETRIEVE, &pInfo->pMidRetriveRes);

×

2121

  QUERY_CHECK_CODE(code, lino, _error);

×

2122

  code = createSpecialDataBlock(STREAM_MID_RETRIEVE, &pInfo->pMidPulloverRes);

×

2123

  QUERY_CHECK_CODE(code, lino, _error);

×

2125

  pInfo->clearState = false;

×

2126

  pInfo->pMidPullDatas = taosArrayInit(4, sizeof(SWinKey));

×

2127

  QUERY_CHECK_NULL(pInfo->pMidPullDatas, code, lino, _error, terrno);

×

2128

  pInfo->pDeletedMap = tSimpleHashInit(4096, hashFn);

×

2129

  QUERY_CHECK_NULL(pInfo->pDeletedMap, code, lino, _error, terrno);

×

2130

  pInfo->destHasPrimaryKey = pIntervalPhyNode->window.destHasPrimaryKey;

×

2131

  pInfo->pOperator = pOperator;

×

2133

  pOperator->operatorType = pPhyNode->type;

×

2134

  if (!IS_FINAL_INTERVAL_OP(pOperator) || numOfChild == 0) {

×

2135

    pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;

×

2137

  pOperator->name = getStreamOpName(pOperator->operatorType);

×

2138

  pOperator->blocking = true;

×

2139

  pOperator->status = OP_NOT_OPENED;

×

2140

  pOperator->info = pInfo;

×

2142

  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {

×

2144

        createOperatorFpSet(NULL, doStreamMidIntervalAggNext, NULL, destroyStreamFinalIntervalOperatorInfo,

×

2148

        createOperatorFpSet(NULL, doStreamFinalIntervalAggNext, NULL, destroyStreamFinalIntervalOperatorInfo,

×

2151

  setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState);

×

2153

  code = initStreamBasicInfo(&pInfo->basic, pOperator);

×

2154

  QUERY_CHECK_CODE(code, lino, _error);

×

2156

  if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL ||

×

2157

      pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL) {

×

2158

    pInfo->basic.primaryPkIndex = -1;

×

2159

    code = initIntervalDownStream(downstream, pPhyNode->type, pInfo, &pInfo->basic);

×

2160

    QUERY_CHECK_CODE(code, lino, _error);

×

2162

  code = appendDownstream(pOperator, &downstream, 1);

×

2163

  QUERY_CHECK_CODE(code, lino, _error);

×

2166

  void*   buff = NULL;

×

2167

  int32_t len = 0;

×

2168

  int32_t res = pAPI->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_CHECKPOINT_NAME,

×

2170

  if (res == TSDB_CODE_SUCCESS) {

×

2171

    doStreamIntervalDecodeOpState(buff, len, pOperator);

×

2172

    taosMemoryFree(buff);

×

2175

  *pOptrInfo = pOperator;

×

2176

  return TSDB_CODE_SUCCESS;

×

2178

_error:

×

2179

  if (pInfo != NULL) destroyStreamFinalIntervalOperatorInfo(pInfo);

×

2180

  destroyOperatorAndDownstreams(pOperator, &downstream, 1);

×

2181

  pTaskInfo->code = code;

×

2182

  return code;

×

2185

void destroyStreamAggSupporter(SStreamAggSupporter* pSup) {

×

2186

  tSimpleHashCleanup(pSup->pResultRows);

×

2187

  destroyDiskbasedBuf(pSup->pResultBuf);

×

2188

  blockDataDestroy(pSup->pScanBlock);

×

2189

  if (pSup->stateStore.streamFileStateDestroy != NULL) {

×

2190

    pSup->stateStore.streamFileStateDestroy(pSup->pState->pFileState);

×

2192

  taosMemoryFreeClear(pSup->pState);

×

2193

  taosMemoryFreeClear(pSup->pDummyCtx);

×

2194

×

2196

void destroyStreamSessionAggOperatorInfo(void* param) {

×

2197

  if (param == NULL) {

×

2198

    return;

×

2200

  SStreamSessionAggOperatorInfo* pInfo = (SStreamSessionAggOperatorInfo*)param;

×

2201

  cleanupBasicInfo(&pInfo->binfo);

×

2202

  if (pInfo->pOperator) {

×

2203

    cleanupResultInfoInStream(pInfo->pOperator->pTaskInfo, pInfo->streamAggSup.pState, &pInfo->pOperator->exprSupp,

×

2205

    pInfo->pOperator = NULL;

×

2208

  destroyStreamBasicInfo(&pInfo->basic);

×

2210

  cleanupExprSupp(&pInfo->scalarSupp);

×

2211

  clearGroupResInfo(&pInfo->groupResInfo);

×

2212

  taosArrayDestroyP(pInfo->pUpdated, destroyFlusedPos);

×

2213

  pInfo->pUpdated = NULL;

×

2214

  destroyStreamAggSupporter(&pInfo->streamAggSup);

×

2216

  if (pInfo->pChildren != NULL) {

×

2217

    int32_t size = taosArrayGetSize(pInfo->pChildren);

×

2218

    for (int32_t i = 0; i < size; i++) {

×

2219

      SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, i);

×

2220

      destroyOperator(pChild);

×

2222

    taosArrayDestroy(pInfo->pChildren);

×

2225

  colDataDestroy(&pInfo->twAggSup.timeWindowData);

×

2226

  blockDataDestroy(pInfo->pDelRes);

×

2227

  blockDataDestroy(pInfo->pWinBlock);

×

2228

  tSimpleHashCleanup(pInfo->pStUpdated);

×

2229

  tSimpleHashCleanup(pInfo->pStDeleted);

×

2230

  cleanupGroupResInfo(&pInfo->groupResInfo);

×

2232

  taosArrayDestroy(pInfo->historyWins);

×

2233

  blockDataDestroy(pInfo->pCheckpointRes);

×

2234

  tSimpleHashCleanup(pInfo->pPkDeleted);

×

2236

  taosMemoryFreeClear(param);

×

2239

int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfCols,

×

2241

  initBasicInfo(pBasicInfo, pResultBlock);

×

2242

  int32_t code = initExprSupp(pSup, pExprInfo, numOfCols, pStore);

×

2243

  if (code != TSDB_CODE_SUCCESS) {

×

2244

    return code;

×

2247

  for (int32_t i = 0; i < numOfCols; ++i) {

×

2248

    pSup->pCtx[i].saveHandle.pBuf = NULL;

×

2251

  return TSDB_CODE_SUCCESS;

×

2254

void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t nums) {

×

2255

  for (int i = 0; i < nums; i++) {

×

2256

    pDummy[i].functionId = pCtx[i].functionId;

×

2257

    pDummy[i].isNotNullFunc = pCtx[i].isNotNullFunc;

×

2258

    pDummy[i].isPseudoFunc = pCtx[i].isPseudoFunc;

×

2259

    pDummy[i].fpSet.init = pCtx[i].fpSet.init;

×

2261

×

2263

int32_t initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex,

×

2265

  SExecTaskInfo* pTaskInfo = downstream->pTaskInfo;

×

2266

  int32_t        code = TSDB_CODE_SUCCESS;

×

2267

  int32_t        lino = 0;

×

2268

  if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) {

×

2269

    SStreamPartitionOperatorInfo* pScanInfo = downstream->info;

×

2270

    pScanInfo->tsColIndex = tsColIndex;

×

2271

    pBasic->primaryPkIndex = pScanInfo->basic.primaryPkIndex;

×

2274

  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {

×

2275

    code = initDownStream(downstream->pDownstream[0], pAggSup, type, tsColIndex, pTwSup, pBasic);

×

2276

    return code;

×

2278

  SStreamScanInfo* pScanInfo = downstream->info;

×

2279

  pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = pAggSup->gap, .parentType = type};

×

2280

  pScanInfo->pState = pAggSup->pState;

×

2281

  if (!pScanInfo->pUpdateInfo) {

×

2282

    code = pAggSup->stateStore.updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, pTwSup->waterMark,

×

2283

                                              pScanInfo->igCheckUpdate, pScanInfo->pkColType, pScanInfo->pkColLen,

×

2284

                                              &pScanInfo->pUpdateInfo);

×

2285

    QUERY_CHECK_CODE(code, lino, _end);

×

2287

  pScanInfo->twAggSup = *pTwSup;

×

2288

  pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo;

×

2289

  if (!hasSrcPrimaryKeyCol(pBasic)) {

×

2290

    pBasic->primaryPkIndex = pScanInfo->basic.primaryPkIndex;

×

2293

_end:

×

2294

  if (code != TSDB_CODE_SUCCESS) {

×

2295

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

2297

  return code;

×

2300

static TSKEY sesionTs(void* pKey) {

×

2301

  SSessionKey* pWinKey = (SSessionKey*)pKey;

×

2302

  return pWinKey->win.skey;

×

2305

int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, int32_t numOfOutput, int64_t gap,

×

2309

  pSup->resultRowSize = (keySize + getResultRowSize(pExpSup->pCtx, numOfOutput)) * ratio;

×

2310

  int32_t lino = 0;

×

2311

  int32_t code = createSpecialDataBlock(STREAM_CLEAR, &pSup->pScanBlock);

×

2312

  QUERY_CHECK_CODE(code, lino, _end);

×

2314

  pSup->gap = gap;

×

2315

  pSup->stateKeySize = keySize;

×

2316

  pSup->stateKeyType = keyType;

×

2317

  pSup->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfOutput, sizeof(SqlFunctionCtx));

×

2318

  QUERY_CHECK_NULL(pSup->pDummyCtx, code, lino, _end, terrno);

×

2320

  pSup->stateStore = *pStore;

×

2321

  pSup->pSessionAPI = pApi;

×

2323

  initDummyFunction(pSup->pDummyCtx, pExpSup->pCtx, numOfOutput);

×

2324

  pSup->pState = taosMemoryCalloc(1, sizeof(SStreamState));

×

2325

  QUERY_CHECK_NULL(pSup->pState, code, lino, _end, terrno);

×

2327

  *(pSup->pState) = *pState;

×

2328

  pSup->stateStore.streamStateSetNumber(pSup->pState, -1, tsIndex);

×

2329

  int32_t funResSize = getMaxFunResSize(pExpSup, numOfOutput);

×

2330

  if (stateType != STREAM_STATE_BUFF_HASH_SORT) {

×

2332

    pSup->pState->pResultRowStore.resultRowGet = getResultRowFromBuf;

×

2333

    pSup->pState->pResultRowStore.resultRowPut = putResultRowToBuf;

×

2334

    pSup->pState->pExprSupp = pExpSup;

×

2337

  if (stateType == STREAM_STATE_BUFF_SORT) {

×

2338

    pSup->pState->pFileState = NULL;

×

2339

    code = pSup->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SSessionKey), pSup->resultRowSize,

×

2340

                                                funResSize, sesionTs, pSup->pState, pTwAggSup->deleteMark, taskIdStr,

×

2341

                                                pHandle->checkpointId, stateType, &pSup->pState->pFileState);

×

2342

  } else if (stateType == STREAM_STATE_BUFF_HASH_SORT || stateType == STREAM_STATE_BUFF_HASH_SEARCH) {

×

2343

    pSup->pState->pFileState = NULL;

×

2344

    code = pSup->stateStore.streamFileStateInit(tsStreamBufferSize, sizeof(SWinKey), pSup->resultRowSize, funResSize,

×

2345

                                                compareTs, pSup->pState, pTwAggSup->deleteMark, taskIdStr,

×

2346

                                                pHandle->checkpointId, stateType, &pSup->pState->pFileState);

×

2348

  QUERY_CHECK_CODE(code, lino, _end);

×

2350

  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

2351

  pSup->pResultRows = tSimpleHashInit(32, hashFn);

×

2352

  QUERY_CHECK_NULL(pSup->pResultRows, code, lino, _end, terrno);

×

2354

  for (int32_t i = 0; i < numOfOutput; ++i) {

×

2355

    pExpSup->pCtx[i].saveHandle.pState = pSup->pState;

×

2358

_end:

×

2359

  if (code != TSDB_CODE_SUCCESS) {

×

2360

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

2362

  return code;

×

2365

bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap) {

×

2366

  if (ts + gap >= pWin->skey && ts - gap <= pWin->ekey) {

×

2367

    return true;

×

2369

  return false;

×

2372

bool isInWindow(SResultWindowInfo* pWinInfo, TSKEY ts, int64_t gap) {

×

2373

  return isInTimeWindow(&pWinInfo->sessionWin.win, ts, gap);

×

2376

void getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,

×

2378

  pKey->win.skey = startTs;

×

2379

  pKey->win.ekey = endTs;

×

2380

  pKey->groupId = groupId;

×

2381

  int32_t code = pAggSup->stateStore.streamStateSessionGetKeyByRange(pAggSup->pState, pKey, pKey);

×

2382

  if (code != TSDB_CODE_SUCCESS) {

×

2383

    SET_SESSION_WIN_KEY_INVALID(pKey);

×

2385

×

2387

bool isInvalidSessionWin(SResultWindowInfo* pWinInfo) { return pWinInfo->sessionWin.win.skey == 0; }

×

2389

bool inWinRange(STimeWindow* range, STimeWindow* cur) {

×

2390

  if (cur->skey >= range->skey && cur->ekey <= range->ekey) {

×

2391

    return true;

×

2393

  return false;

×

2396

void clearOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) { pAPI->streamStateClearBuff(pState, pPos); }

×

2398

int32_t setSessionOutputBuf(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,

×

2400

  int32_t code = TSDB_CODE_SUCCESS;

×

2401

  int32_t lino = 0;

×

2402

  pCurWin->sessionWin.groupId = groupId;

×

2403

  pCurWin->sessionWin.win.skey = startTs;

×

2404

  pCurWin->sessionWin.win.ekey = endTs;

×

2405

  int32_t size = pAggSup->resultRowSize;

×

2406

  code = pAggSup->stateStore.streamStateSessionAddIfNotExist(pAggSup->pState, &pCurWin->sessionWin, pAggSup->gap,

×

2407

                                                             (void**)&pCurWin->pStatePos, &size, pWinCode);

×

2408

  QUERY_CHECK_CODE(code, lino, _end);

×

2410

  if (*pWinCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->sessionWin.win)) {

×

2411

    *pWinCode = TSDB_CODE_FAILED;

×

2412

    clearOutputBuf(pAggSup->pState, pCurWin->pStatePos, &pAggSup->pSessionAPI->stateStore);

×

2415

  if (*pWinCode == TSDB_CODE_SUCCESS) {

×

2416

    pCurWin->isOutput = true;

×

2417

    if (pCurWin->pStatePos->needFree) {

×

2418

      pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->sessionWin);

×

2421

    pCurWin->sessionWin.win.skey = startTs;

×

2422

    pCurWin->sessionWin.win.ekey = endTs;

×

2424

  qDebug("===stream===set session window buff .start:%" PRId64 ",end:%" PRId64 ",groupid:%" PRIu64,

×

2427

_end:

×

2428

  if (code != TSDB_CODE_SUCCESS) {

×

2429

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

2431

  return code;

×

2434

void getSessionWinBuf(SStreamAggSupporter* pAggSup, SStreamStateCur* pCur, SResultWindowInfo* pWinInfo,

×

2436

  int32_t size = 0;

×

2437

  (*pWinCode) = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pWinInfo->sessionWin,

×

2438

                                                                 (void**)&pWinInfo->pStatePos, &size);

×

2439

  if ((*pWinCode) != TSDB_CODE_SUCCESS) {

×

2440

    return;

×

2443

  pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pCur);

×

2446

int32_t saveDeleteInfo(SArray* pWins, SSessionKey key) {

×

2447

  int32_t code = TSDB_CODE_SUCCESS;

×

2448

  int32_t lino = 0;

×

2449

  void*   res = taosArrayPush(pWins, &key);

×

2450

  if (!res) {

×

2451

    code = terrno;

×

2452

    QUERY_CHECK_CODE(code, lino, _end);

×

2455

_end:

×

2456

  if (code != TSDB_CODE_SUCCESS) {

×

2457

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

2459

  return code;

×

2462

int32_t saveDeleteRes(SSHashObj* pStDelete, SSessionKey key) {

×

2463

  key.win.ekey = key.win.skey;

×

2464

  return tSimpleHashPut(pStDelete, &key, sizeof(SSessionKey), NULL, 0);

×

2467

void releaseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) {

×

2468

  pAPI->streamStateReleaseBuf(pState, pPos, false);

×

2469

×

2471

void removeSessionResult(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey* pKey) {

×

2472

  int32_t     code = TSDB_CODE_SUCCESS;

×

2473

  int32_t     lino = 0;

×

2474

  SSessionKey key = {0};

×

2475

  getSessionHashKey(pKey, &key);

×

2476

  void* pVal = tSimpleHashGet(pHashMap, &key, sizeof(SSessionKey));

×

2477

  if (pVal) {

×

2478

    releaseOutputBuf(pAggSup->pState, *(void**)pVal, &pAggSup->pSessionAPI->stateStore);

×

2479

    int32_t tmpRes = tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey));

×

2480

    qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

2482

  int32_t tmpRes = tSimpleHashRemove(pResMap, &key, sizeof(SSessionKey));

×

2483

  qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

2484

×

2486

void getSessionHashKey(const SSessionKey* pKey, SSessionKey* pHashKey) {

×

2487

  *pHashKey = *pKey;

×

2488

  pHashKey->win.ekey = pKey->win.skey;

×

2489

×

2491

void removeSessionDeleteResults(SSHashObj* pHashMap, SArray* pWins) {

×

2492

  if (tSimpleHashGetSize(pHashMap) == 0) {

×

2493

    return;

×

2495

  int32_t size = taosArrayGetSize(pWins);

×

2496

  for (int32_t i = 0; i < size; i++) {

×

2497

    SResultWindowInfo* pWin = taosArrayGet(pWins, i);

×

2498

    if (!pWin) continue;

×

2499

    SSessionKey key = {0};

×

2500

    getSessionHashKey(&pWin->sessionWin, &key);

×

2501

    int32_t tmpRes = tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey));

×

2502

    qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

2506

void removeSessionResults(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SArray* pWins) {

×

2507

  if (tSimpleHashGetSize(pHashMap) == 0) {

×

2508

    return;

×

2510

  int32_t size = taosArrayGetSize(pWins);

×

2511

  for (int32_t i = 0; i < size; i++) {

×

2512

    SSessionKey* pWin = taosArrayGet(pWins, i);

×

2513

    if (!pWin) continue;

×

2514

    SSessionKey key = {0};

×

2515

    getSessionHashKey(pWin, &key);

×

2516

    void* pVal = tSimpleHashGet(pHashMap, &key, sizeof(SSessionKey));

×

2517

    if (pVal) {

×

2518

      releaseOutputBuf(pAggSup->pState, *(void**)pVal, &pAggSup->pSessionAPI->stateStore);

×

2519

      int32_t tmpRes = tSimpleHashRemove(pHashMap, &key, sizeof(SSessionKey));

×

2520

      qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

2525

int32_t updateSessionWindowInfo(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo, TSKEY* pStartTs,

×

2529

  int32_t code = TSDB_CODE_SUCCESS;

×

2530

  int32_t lino = 0;

×

2531

  for (int32_t i = start; i < rows; ++i) {

×

2532

    if (!isInWindow(pWinInfo, pStartTs[i], gap) && (!pEndTs || !isInWindow(pWinInfo, pEndTs[i], gap))) {

×

2533

      (*pWinRos) = i - start;

×

2534

      goto _end;

×

2536

    if (pWinInfo->sessionWin.win.skey > pStartTs[i]) {

×

2537

      if (pStDeleted && pWinInfo->isOutput) {

×

2538

        code = saveDeleteRes(pStDeleted, pWinInfo->sessionWin);

×

2539

        QUERY_CHECK_CODE(code, lino, _end);

×

2541

      removeSessionResult(pAggSup, pStUpdated, pResultRows, &pWinInfo->sessionWin);

×

2542

      pWinInfo->sessionWin.win.skey = pStartTs[i];

×

2544

    pWinInfo->sessionWin.win.ekey = TMAX(pWinInfo->sessionWin.win.ekey, pStartTs[i]);

×

2545

    if (pEndTs) {

×

2546

      pWinInfo->sessionWin.win.ekey = TMAX(pWinInfo->sessionWin.win.ekey, pEndTs[i]);

×

2548

    memcpy(pWinInfo->pStatePos->pKey, &pWinInfo->sessionWin, sizeof(SSessionKey));

×

2550

  (*pWinRos) = rows - start;

×

2552

_end:

×

2553

  if (code != TSDB_CODE_SUCCESS) {

×

2554

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

2556

  return code;

×

2559

static int32_t initSessionOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pResult, SqlFunctionCtx* pCtx,

×

2561

  *pResult = (SResultRow*)pWinInfo->pStatePos->pRowBuff;

×

2563

  (*pResult)->win = pWinInfo->sessionWin.win;

×

2564

  return setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowEntryInfoOffset);

×

2567

int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindowInfo* pCurWin, SResultRow** pResult,

×

2570

  int32_t        code = TSDB_CODE_SUCCESS;

×

2571

  int32_t        lino = 0;

×

2572

  SExprSupp*     pSup = &pOperator->exprSupp;

×

2573

  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

×

2574

  code = initSessionOutputBuf(pCurWin, pResult, pSup->pCtx, numOutput, pSup->rowEntryInfoOffset);

×

2575

  QUERY_CHECK_CODE(code, lino, _end);

×

2577

  updateTimeWindowInfo(pTimeWindowData, &pCurWin->sessionWin.win, winDelta);

×

2578

  code = applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, pTimeWindowData, startIndex, winRows, rows, numOutput);

×

2580

_end:

×

2581

  if (code != TSDB_CODE_SUCCESS) {

×

2582

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

2584

  return code;

×

2587

void doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKey) {

×

2588

  pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, pKey);

×

2589

  SSessionKey hashKey = {0};

×

2590

  getSessionHashKey(pKey, &hashKey);

×

2591

  int32_t tmpRes = tSimpleHashRemove(pAggSup->pResultRows, &hashKey, sizeof(SSessionKey));

×

2592

  qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

2593

×

2595

void setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo* pWinInfo) {

×

2596

  void* pVal = tSimpleHashGet(pStUpdated, &pWinInfo->sessionWin, sizeof(SSessionKey));

×

2597

  if (pVal) {

×

2598

    SResultWindowInfo* pWin = pVal;

×

2599

    pWinInfo->isOutput = pWin->isOutput;

×

2601

×

2603

void getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin,

×

2605

  SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->sessionWin);

×

2606

  pNextWin->isOutput = true;

×

2607

  setSessionWinOutputInfo(pStUpdated, pNextWin);

×

2608

  int32_t size = 0;

×

2609

  pNextWin->sessionWin = pCurWin->sessionWin;

×

2610

  int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->sessionWin,

×

2611

                                                                  (void**)&pNextWin->pStatePos, &size);

×

2612

  if (code != TSDB_CODE_SUCCESS) {

×

2613

    SET_SESSION_WIN_INVALID(*pNextWin);

×

2615

  pAggSup->stateStore.streamStateFreeCur(pCur);

×

2616

×

2618

int32_t compactTimeWindow(SExprSupp* pSup, SStreamAggSupporter* pAggSup, STimeWindowAggSupp* pTwAggSup,

×

2621

  int32_t     code = TSDB_CODE_SUCCESS;

×

2622

  int32_t     lino = 0;

×

2623

  SResultRow* pCurResult = NULL;

×

2624

  int32_t     numOfOutput = pSup->numOfExprs;

×

2625

  code = initSessionOutputBuf(pCurWin, &pCurResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset);

×

2626

  QUERY_CHECK_CODE(code, lino, _end);

×

2628

  SResultRow* pWinResult = NULL;

×

2629

  code = initSessionOutputBuf(pNextWin, &pWinResult, pAggSup->pDummyCtx, numOfOutput, pSup->rowEntryInfoOffset);

×

2630

  QUERY_CHECK_CODE(code, lino, _end);

×

2632

  pCurWin->sessionWin.win.ekey = TMAX(pCurWin->sessionWin.win.ekey, pNextWin->sessionWin.win.ekey);

×

2633

  memcpy(pCurWin->pStatePos->pKey, &pCurWin->sessionWin, sizeof(SSessionKey));

×

2635

  int64_t winDelta = 0;

×

2636

  if (addGap) {

×

2637

    winDelta = pAggSup->gap;

×

2639

  updateTimeWindowInfo(&pTwAggSup->timeWindowData, &pCurWin->sessionWin.win, winDelta);

×

2640

  code = compactFunctions(pSup->pCtx, pAggSup->pDummyCtx, numOfOutput, pTaskInfo, &pTwAggSup->timeWindowData);

×

2641

  QUERY_CHECK_CODE(code, lino, _end);

×

2643

  int32_t tmpRes = tSimpleHashRemove(pStUpdated, &pNextWin->sessionWin, sizeof(SSessionKey));

×

2644

  qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

2646

  if (pNextWin->isOutput && pStDeleted) {

×

2647

    qDebug("===stream=== save delete window info %" PRId64 ", %" PRIu64, pNextWin->sessionWin.win.skey,

×

2649

    code = saveDeleteRes(pStDeleted, pNextWin->sessionWin);

×

2650

    QUERY_CHECK_CODE(code, lino, _end);

×

2652

  removeSessionResult(pAggSup, pStUpdated, pAggSup->pResultRows, &pNextWin->sessionWin);

×

2653

  doDeleteSessionWindow(pAggSup, &pNextWin->sessionWin);

×

2654

  releaseOutputBuf(pAggSup->pState, pNextWin->pStatePos, &pAggSup->pSessionAPI->stateStore);

×

2656

_end:

×

2657

  if (code != TSDB_CODE_SUCCESS) {

×

2658

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

2660

  return code;

×

2663

static int32_t compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SSHashObj* pStUpdated,

×

2665

  int32_t                        code = TSDB_CODE_SUCCESS;

×

2666

  int32_t                        lino = 0;

×

2667

  SExprSupp*                     pSup = &pOperator->exprSupp;

×

2668

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

2669

  SStorageAPI*                   pAPI = &pOperator->pTaskInfo->storageAPI;

×

2670

  int32_t                        winNum = 0;

×

2671

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

2672

  SResultRow*                    pCurResult = NULL;

×

2673

  int32_t                        numOfOutput = pOperator->exprSupp.numOfExprs;

×

2674

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

2677

  while (1) {

×

2678

    SResultWindowInfo winInfo = {0};

×

2679

    getNextSessionWinInfo(pAggSup, pStUpdated, pCurWin, &winInfo);

×

2680

    if (!IS_VALID_SESSION_WIN(winInfo) || !isInWindow(pCurWin, winInfo.sessionWin.win.skey, pAggSup->gap) ||

×

2681

        !inWinRange(&pAggSup->winRange, &winInfo.sessionWin.win)) {

×

2682

      releaseOutputBuf(pAggSup->pState, winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);

×

2683

      break;

×

2686

        compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, pCurWin, &winInfo, pStUpdated, pStDeleted, true);

×

2687

    QUERY_CHECK_CODE(code, lino, _end);

×

2688

    winNum++;

×

2690

  if (pWinNum) {

×

2691

    (*pWinNum) = winNum;

×

2694

_end:

×

2695

  if (code != TSDB_CODE_SUCCESS) {

×

2696

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

2698

  return code;

×

2701

static void compactSessionSemiWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin) {

×

2702

  SExprSupp*                     pSup = &pOperator->exprSupp;

×

2703

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

2704

  SStorageAPI*                   pAPI = &pOperator->pTaskInfo->storageAPI;

×

2705

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

2706

  SResultRow*                    pCurResult = NULL;

×

2707

  int32_t                        numOfOutput = pOperator->exprSupp.numOfExprs;

×

2708

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

2710

  while (1) {

×

2711

    SResultWindowInfo winInfo = {0};

×

2712

    getNextSessionWinInfo(pAggSup, NULL, pCurWin, &winInfo);

×

2713

    if (!IS_VALID_SESSION_WIN(winInfo) || !isInWindow(pCurWin, winInfo.sessionWin.win.skey, pAggSup->gap) ||

×

2714

        !inWinRange(&pAggSup->winRange, &winInfo.sessionWin.win)) {

×

2715

      releaseOutputBuf(pAggSup->pState, winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);

×

2716

      break;

×

2718

    pCurWin->sessionWin.win.ekey = TMAX(pCurWin->sessionWin.win.ekey, winInfo.sessionWin.win.ekey);

×

2719

    memcpy(pCurWin->pStatePos->pKey, &pCurWin->sessionWin, sizeof(SSessionKey));

×

2720

    doDeleteSessionWindow(pAggSup, &winInfo.sessionWin);

×

2721

    releaseOutputBuf(pAggSup->pState, winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);

×

2723

×

2725

int32_t saveSessionOutputBuf(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo) {

×

2726

  qDebug("===stream===try save session result skey:%" PRId64 ", ekey:%" PRId64 ".pos%d", pWinInfo->sessionWin.win.skey,

×

2728

  return pAggSup->stateStore.streamStateSessionPut(pAggSup->pState, &pWinInfo->sessionWin, pWinInfo->pStatePos,

×

2732

static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pStUpdated,

×

2734

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

2735

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

2736

  int32_t                        numOfOutput = pOperator->exprSupp.numOfExprs;

×

2737

  uint64_t                       groupId = pSDataBlock->info.id.groupId;

×

2738

  int32_t                        code = TSDB_CODE_SUCCESS;

×

2739

  int32_t                        lino = 0;

×

2740

  SResultRow*                    pResult = NULL;

×

2741

  int32_t                        rows = pSDataBlock->info.rows;

×

2742

  int32_t                        winRows = 0;

×

2743

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

2745

  pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);

×

2746

  pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;

×

2747

  if (pAggSup->winRange.ekey <= 0) {

×

2748

    pAggSup->winRange.ekey = INT64_MAX;

×

2751

  SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);

×

2752

  TSKEY*           startTsCols = (int64_t*)pStartTsCol->pData;

×

2753

  SColumnInfoData* pEndTsCol = NULL;

×

2754

  if (hasEndTs) {

×

2755

    pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->endTsIndex);

×

2757

    pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);

×

2760

  TSKEY* endTsCols = (int64_t*)pEndTsCol->pData;

×

2762

  void*            pPkVal = NULL;

×

2763

  int32_t          pkLen = 0;

×

2764

  SColumnInfoData* pPkColDataInfo = NULL;

×

2765

  if (hasSrcPrimaryKeyCol(&pInfo->basic)) {

×

2766

    pPkColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->basic.primaryPkIndex);

×

2769

  for (int32_t i = 0; i < rows;) {

×

2770

    if (hasSrcPrimaryKeyCol(&pInfo->basic) && !IS_FINAL_SESSION_OP(pOperator) && pInfo->ignoreExpiredData) {

×

2771

      pPkVal = colDataGetData(pPkColDataInfo, i);

×

2772

      pkLen = colDataGetRowLength(pPkColDataInfo, i);

×

2774

    if (!IS_FINAL_SESSION_OP(pOperator) && pInfo->ignoreExpiredData &&

×

2775

        checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo, &pInfo->twAggSup,

×

2776

                         pSDataBlock->info.id.uid, endTsCols[i], pPkVal, pkLen)) {

×

2777

      i++;

×

2778

      continue;

×

2780

    SResultWindowInfo winInfo = {0};

×

2781

    int32_t           winCode = TSDB_CODE_SUCCESS;

×

2782

    code = setSessionOutputBuf(pAggSup, startTsCols[i], endTsCols[i], groupId, &winInfo, &winCode);

×

2783

    QUERY_CHECK_CODE(code, lino, _end);

×

2785

    if (winCode != TSDB_CODE_SUCCESS && IS_NORMAL_SESSION_OP(pOperator) &&

×

2786

        BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {

×

2787

      code = addSessionAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &winInfo.sessionWin, &pInfo->basic.notifyEventSup,

×

2789

      QUERY_CHECK_CODE(code, lino, _end);

×

2793

    if (!winInfo.pStatePos) {

×

2794

      continue;

×

2796

    setSessionWinOutputInfo(pStUpdated, &winInfo);

×

2797

    code = updateSessionWindowInfo(pAggSup, &winInfo, startTsCols, endTsCols, groupId, rows, i, pAggSup->gap,

×

2799

    QUERY_CHECK_CODE(code, lino, _end);

×

2801

    int64_t winDelta = 0;

×

2802

    if (addGap) {

×

2803

      winDelta = pAggSup->gap;

×

2805

    code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &winInfo, &pResult, i, winRows, rows, numOfOutput,

×

2807

    QUERY_CHECK_CODE(code, lino, _end);

×

2809

    code = compactSessionWindow(pOperator, &winInfo, pStUpdated, pStDeleted, addGap, NULL);

×

2810

    QUERY_CHECK_CODE(code, lino, _end);

×

2812

    code = saveSessionOutputBuf(pAggSup, &winInfo);

×

2813

    QUERY_CHECK_CODE(code, lino, _end);

×

2815

    if (pInfo->destHasPrimaryKey && winInfo.isOutput && IS_NORMAL_SESSION_OP(pOperator)) {

×

2816

      code = saveDeleteRes(pInfo->pPkDeleted, winInfo.sessionWin);

×

2817

      QUERY_CHECK_CODE(code, lino, _end);

×

2819

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) {

×

2820

      code = saveResult(winInfo, pStUpdated);

×

2821

      QUERY_CHECK_CODE(code, lino, _end);

×

2823

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

2824

      winInfo.pStatePos->beUpdated = true;

×

2825

      SSessionKey key = {0};

×

2826

      getSessionHashKey(&winInfo.sessionWin, &key);

×

2827

      code = tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo));

×

2828

      QUERY_CHECK_CODE(code, lino, _end);

×

2831

    i += winRows;

×

2834

_end:

×

2835

  if (code != TSDB_CODE_SUCCESS) {

×

2836

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

2838

×

2840

int32_t doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result) {

×

2841

  int32_t          code = TSDB_CODE_SUCCESS;

×

2842

  int32_t          lino = 0;

×

2843

  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);

×

2844

  TSKEY*           startDatas = (TSKEY*)pStartTsCol->pData;

×

2845

  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);

×

2846

  TSKEY*           endDatas = (TSKEY*)pEndTsCol->pData;

×

2847

  SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);

×

2848

  uint64_t*        gpDatas = (uint64_t*)pGroupCol->pData;

×

2849

  for (int32_t i = 0; i < pBlock->info.rows; i++) {

×

2850

    while (1) {

×

2851

      SSessionKey curWin = {0};

×

2852

      getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], &curWin);

×

2853

      if (IS_INVALID_SESSION_WIN_KEY(curWin)) {

×

2854

        break;

×

2856

      doDeleteSessionWindow(pAggSup, &curWin);

×

2857

      if (result) {

×

2858

        code = saveDeleteInfo(result, curWin);

×

2859

        QUERY_CHECK_CODE(code, lino, _end);

×

2864

_end:

×

2865

  if (code != TSDB_CODE_SUCCESS) {

×

2866

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

2868

  return code;

×

2871

inline int32_t sessionKeyCompareAsc(const void* pKey1, const void* pKey2) {

×

2872

  SResultWindowInfo* pWinInfo1 = (SResultWindowInfo*)pKey1;

×

2873

  SResultWindowInfo* pWinInfo2 = (SResultWindowInfo*)pKey2;

×

2874

  SSessionKey*       pWin1 = &pWinInfo1->sessionWin;

×

2875

  SSessionKey*       pWin2 = &pWinInfo2->sessionWin;

×

2877

  if (pWin1->groupId > pWin2->groupId) {

×

2878

    return 1;

×

2879

  } else if (pWin1->groupId < pWin2->groupId) {

×

2880

    return -1;

×

2883

  if (pWin1->win.skey > pWin2->win.skey) {

×

2884

    return 1;

×

2885

  } else if (pWin1->win.skey < pWin2->win.skey) {

×

2886

    return -1;

×

2889

  return 0;

×

2892

static int32_t appendToDeleteDataBlock(SOperatorInfo* pOp, SSDataBlock *pBlock, SSessionKey *pKey) {

×

2893

  int32_t        code = TSDB_CODE_SUCCESS;

×

2894

  int32_t        lino = 0;

×

2895

  SExecTaskInfo* pTaskInfo = pOp->pTaskInfo;

×

2897

  QUERY_CHECK_NULL(pBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);

×

2898

  QUERY_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA);

×

2900

  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);

×

2901

  code = colDataSetVal(pStartTsCol, pBlock->info.rows, (const char*)&pKey->win.skey, false);

×

2902

  QUERY_CHECK_CODE(code, lino, _end);

×

2904

  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);

×

2905

  code = colDataSetVal(pEndTsCol, pBlock->info.rows, (const char*)&pKey->win.skey, false);

×

2906

  QUERY_CHECK_CODE(code, lino, _end);

×

2908

  SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);

×

2909

  colDataSetNULL(pUidCol, pBlock->info.rows);

×

2911

  SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);

×

2912

  code = colDataSetVal(pGpCol, pBlock->info.rows, (const char*)&pKey->groupId, false);

×

2913

  QUERY_CHECK_CODE(code, lino, _end);

×

2915

  SColumnInfoData* pCalStCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);

×

2916

  colDataSetNULL(pCalStCol, pBlock->info.rows);

×

2918

  SColumnInfoData* pCalEdCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);

×

2919

  colDataSetNULL(pCalEdCol, pBlock->info.rows);

×

2921

  SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);

×

2922

  if (!pTableCol) {

×

2923

    QUERY_CHECK_CODE(code, lino, _end);

×

2926

  void*        tbname = NULL;

×

2927

  int32_t      winCode = TSDB_CODE_SUCCESS;

×

2928

  SStorageAPI* pAPI = &pOp->pTaskInfo->storageAPI;

×

2930

      pAPI->stateStore.streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, pKey->groupId, &tbname, false, &winCode);

×

2931

  QUERY_CHECK_CODE(code, lino, _end);

×

2933

  if (winCode != TSDB_CODE_SUCCESS) {

×

2934

    colDataSetNULL(pTableCol, pBlock->info.rows);

×

2937

    STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));

×

2938

    code = colDataSetVal(pTableCol, pBlock->info.rows, (const char*)parTbName, false);

×

2939

    QUERY_CHECK_CODE(code, lino, _end);

×

2940

    pAPI->stateStore.streamStateFreeVal(tbname);

×

2942

  pBlock->info.rows += 1;

×

2944

_end:

×

2945

  if (code != TSDB_CODE_SUCCESS) {

×

2946

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

2948

  return code;

×

2951

void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite,

×

2953

  int32_t        code = TSDB_CODE_SUCCESS;

×

2954

  int32_t        lino = 0;

×

2955

  SExecTaskInfo* pTaskInfo = pOp->pTaskInfo;

×

2956

  int64_t        minWindowSize = getMinWindowSize(pOp);

×

2957

  int32_t        numOfRows = getNumOfTotalRes(pGroupResInfo);

×

2959

  blockDataCleanup(pBlock);

×

2960

  int32_t size = tSimpleHashGetSize(pStDeleted);

×

2961

  if (minWindowSize > 0) {

×

2963

    for (int32_t i = pGroupResInfo->delIndex; i < numOfRows; ++i) {

×

2964

      SResultWindowInfo* pWinInfo = taosArrayGet(pGroupResInfo->pRows, i);

×

2965

      SRowBuffPos*       pPos = pWinInfo->pStatePos;

×

2966

      SSessionKey*       pKey = (SSessionKey*)pPos->pKey;

×

2967

      if (pKey->win.ekey - pKey->win.skey < minWindowSize) {

×

2968

        size++;

×

2972

  if (size == 0) {

×

2973

    return;

×

2975

  code = blockDataEnsureCapacity(pBlock, size);

×

2976

  QUERY_CHECK_CODE(code, lino, _end);

×

2978

  int32_t iter = 0;

×

2979

  while (((*Ite) = tSimpleHashIterate(pStDeleted, *Ite, &iter)) != NULL) {

×

2980

    if (pBlock->info.rows + 1 > pBlock->info.capacity) {

×

2981

      break;

×

2983

    SSessionKey*     res = tSimpleHashGetKey(*Ite, NULL);

×

2984

    code = appendToDeleteDataBlock(pOp, pBlock, res);

×

2985

    QUERY_CHECK_CODE(code, lino, _end);

×

2988

  if (minWindowSize > 0) {

×

2989

    for (int32_t i = pGroupResInfo->delIndex; i < numOfRows; ++i) {

×

2990

      SResultWindowInfo* pWinInfo = taosArrayGet(pGroupResInfo->pRows, i);

×

2991

      SRowBuffPos*       pPos = pWinInfo->pStatePos;

×

2992

      SSessionKey*       pKey = (SSessionKey*)pPos->pKey;

×

2993

      if (pKey->win.ekey - pKey->win.skey < minWindowSize) {

×

2994

        code = appendToDeleteDataBlock(pOp, pBlock, pKey);

×

2995

        QUERY_CHECK_CODE(code, lino, _end);

×

2998

    pGroupResInfo->delIndex = numOfRows;

×

3001

_end:

×

3002

  if ((*Ite) == NULL) {

×

3003

    tSimpleHashClear(pStDeleted);

×

3006

  if (code != TSDB_CODE_SUCCESS) {

×

3007

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3011

static int32_t rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SSHashObj* pStUpdated) {

×

3012

  int32_t        winCode = TSDB_CODE_SUCCESS;

×

3013

  int32_t        code = TSDB_CODE_SUCCESS;

×

3014

  int32_t        lino = 0;

×

3015

  SExprSupp*     pSup = &pOperator->exprSupp;

×

3016

  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

×

3017

  SStorageAPI*   pAPI = &pOperator->pTaskInfo->storageAPI;

×

3019

  int32_t                        size = taosArrayGetSize(pWinArray);

×

3020

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3021

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

3022

  int32_t                        numOfOutput = pSup->numOfExprs;

×

3023

  int32_t                        numOfChild = taosArrayGetSize(pInfo->pChildren);

×

3025

  for (int32_t i = 0; i < size; i++) {

×

3026

    SSessionKey*      pWinKey = taosArrayGet(pWinArray, i);

×

3027

    int32_t           num = 0;

×

3028

    SResultWindowInfo parentWin = {0};

×

3029

    for (int32_t j = 0; j < numOfChild; j++) {

×

3030

      SOperatorInfo*                 pChild = taosArrayGetP(pInfo->pChildren, j);

×

3031

      SStreamSessionAggOperatorInfo* pChInfo = pChild->info;

×

3032

      SStreamAggSupporter*           pChAggSup = &pChInfo->streamAggSup;

×

3033

      SSessionKey                    chWinKey = {0};

×

3034

      getSessionHashKey(pWinKey, &chWinKey);

×

3035

      SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentNext(pChAggSup->pState, &chWinKey);

×

3036

      SResultRow*      pResult = NULL;

×

3037

      SResultRow*      pChResult = NULL;

×

3038

      while (1) {

×

3039

        SResultWindowInfo childWin = {0};

×

3040

        childWin.sessionWin = *pWinKey;

×

3041

        getSessionWinBuf(pChAggSup, pCur, &childWin, &winCode);

×

3043

        if (winCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &childWin.sessionWin.win)) {

×

3044

          releaseOutputBuf(pAggSup->pState, childWin.pStatePos, &pAggSup->stateStore);

×

3045

          continue;

×

3048

        if (winCode == TSDB_CODE_SUCCESS && inWinRange(&pWinKey->win, &childWin.sessionWin.win)) {

×

3049

          if (num == 0) {

×

3050

            code = setSessionOutputBuf(pAggSup, pWinKey->win.skey, pWinKey->win.ekey, pWinKey->groupId, &parentWin,

×

3052

            QUERY_CHECK_CODE(code, lino, _end);

×

3054

            parentWin.sessionWin = childWin.sessionWin;

×

3055

            memcpy(parentWin.pStatePos->pKey, &parentWin.sessionWin, sizeof(SSessionKey));

×

3056

            code = initSessionOutputBuf(&parentWin, &pResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset);

×

3057

            QUERY_CHECK_CODE(code, lino, _end);

×

3059

          num++;

×

3060

          parentWin.sessionWin.win.skey = TMIN(parentWin.sessionWin.win.skey, childWin.sessionWin.win.skey);

×

3061

          parentWin.sessionWin.win.ekey = TMAX(parentWin.sessionWin.win.ekey, childWin.sessionWin.win.ekey);

×

3062

          memcpy(parentWin.pStatePos->pKey, &parentWin.sessionWin, sizeof(SSessionKey));

×

3064

          updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin.sessionWin.win, pAggSup->gap);

×

3065

          code = initSessionOutputBuf(&childWin, &pChResult, pChild->exprSupp.pCtx, numOfOutput,

×

3067

          QUERY_CHECK_CODE(code, lino, _end);

×

3069

          code = compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo,

×

3071

          QUERY_CHECK_CODE(code, lino, _end);

×

3073

          code = compactSessionWindow(pOperator, &parentWin, pStUpdated, NULL, true, NULL);

×

3074

          QUERY_CHECK_CODE(code, lino, _end);

×

3076

          releaseOutputBuf(pAggSup->pState, childWin.pStatePos, &pAggSup->stateStore);

×

3078

          releaseOutputBuf(pAggSup->pState, childWin.pStatePos, &pAggSup->stateStore);

×

3079

          break;

×

3082

      pAPI->stateStore.streamStateFreeCur(pCur);

×

3084

    if (num > 0) {

×

3085

      code = saveResult(parentWin, pStUpdated);

×

3086

      QUERY_CHECK_CODE(code, lino, _end);

×

3088

      code = saveSessionOutputBuf(pAggSup, &parentWin);

×

3089

      QUERY_CHECK_CODE(code, lino, _end);

×

3093

_end:

×

3094

  if (code != TSDB_CODE_SUCCESS) {

×

3095

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3097

  return code;

×

3100

int32_t closeSessionWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SSHashObj* pClosed) {

×

3101

  int32_t code = TSDB_CODE_SUCCESS;

×

3102

  int32_t lino = 0;

×

3103

  void*   pIte = NULL;

×

3104

  int32_t iter = 0;

×

3105

  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {

×

3106

    SResultWindowInfo* pWinInfo = pIte;

×

3107

    if (isCloseWindow(&pWinInfo->sessionWin.win, pTwSup)) {

×

3108

      if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE && pClosed) {

×

3109

        code = saveResult(*pWinInfo, pClosed);

×

3110

        QUERY_CHECK_CODE(code, lino, _end);

×

3112

      SSessionKey* pKey = tSimpleHashGetKey(pIte, NULL);

×

3113

      code = tSimpleHashIterateRemove(pHashMap, pKey, sizeof(SSessionKey), &pIte, &iter);

×

3114

      QUERY_CHECK_CODE(code, lino, _end);

×

3117

_end:

×

3118

  if (code != TSDB_CODE_SUCCESS) {

×

3119

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3121

  return code;

×

3124

static int32_t closeChildSessionWindow(SArray* pChildren, TSKEY maxTs) {

×

3125

  int32_t code = TSDB_CODE_SUCCESS;

×

3126

  int32_t lino = 0;

×

3128

  int32_t size = taosArrayGetSize(pChildren);

×

3129

  for (int32_t i = 0; i < size; i++) {

×

3130

    SOperatorInfo*                 pChildOp = taosArrayGetP(pChildren, i);

×

3131

    SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info;

×

3132

    pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs);

×

3133

    code = closeSessionWindow(pChInfo->streamAggSup.pResultRows, &pChInfo->twAggSup, NULL);

×

3134

    QUERY_CHECK_CODE(code, lino, _end);

×

3136

_end:

×

3137

  if (code != TSDB_CODE_SUCCESS) {

×

3138

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3140

  return code;

×

3143

int32_t getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated) {

×

3144

  int32_t code = TSDB_CODE_SUCCESS;

×

3145

  int32_t lino = 0;

×

3146

  void*   pIte = NULL;

×

3147

  int32_t iter = 0;

×

3148

  while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {

×

3149

    SResultWindowInfo* pWinInfo = pIte;

×

3150

    if (!pWinInfo->pStatePos->beUpdated) {

×

3151

      continue;

×

3153

    pWinInfo->pStatePos->beUpdated = false;

×

3154

    code = saveResult(*pWinInfo, pStUpdated);

×

3155

    QUERY_CHECK_CODE(code, lino, _end);

×

3158

_end:

×

3159

  if (code != TSDB_CODE_SUCCESS) {

×

3160

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3162

  return code;

×

3165

int32_t copyDeleteWindowInfo(SArray* pResWins, SSHashObj* pStDeleted) {

×

3166

  int32_t code = TSDB_CODE_SUCCESS;

×

3167

  int32_t lino = 0;

×

3168

  int32_t size = taosArrayGetSize(pResWins);

×

3169

  for (int32_t i = 0; i < size; i++) {

×

3170

    SSessionKey* pWinKey = taosArrayGet(pResWins, i);

×

3171

    if (!pWinKey) continue;

×

3172

    SSessionKey winInfo = {0};

×

3173

    getSessionHashKey(pWinKey, &winInfo);

×

3174

    code = tSimpleHashPut(pStDeleted, &winInfo, sizeof(SSessionKey), NULL, 0);

×

3175

    QUERY_CHECK_CODE(code, lino, _end);

×

3178

_end:

×

3179

  if (code != TSDB_CODE_SUCCESS) {

×

3180

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3182

  return code;

×

3186

void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList) {

×

3187

  pGroupResInfo->pRows = pArrayList;

×

3188

  pGroupResInfo->index = 0;

×

3189

  pGroupResInfo->pBuf = NULL;

×

3190

  pGroupResInfo->freeItem = false;

×

3191

  pGroupResInfo->delIndex = 0;

×

3192

×

3194

int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDataBlock* pBlock, SExprSupp* pSup,

×

3196

  int32_t         code = TSDB_CODE_SUCCESS;

×

3197

  int32_t         lino = 0;

×

3198

  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;

×

3199

  SStorageAPI*    pAPI = &pTaskInfo->storageAPI;

×

3200

  SExprInfo*      pExprInfo = pSup->pExprInfo;

×

3201

  int32_t         numOfExprs = pSup->numOfExprs;

×

3202

  int32_t*        rowEntryOffset = pSup->rowEntryInfoOffset;

×

3203

  SqlFunctionCtx* pCtx = pSup->pCtx;

×

3204

  int64_t         minWindowSize = getMinWindowSize(pOperator);

×

3206

  int32_t numOfRows = getNumOfTotalRes(pGroupResInfo);

×

3208

  for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) {

×

3209

    SResultWindowInfo* pWinInfo = taosArrayGet(pGroupResInfo->pRows, i);

×

3210

    SRowBuffPos*       pPos = pWinInfo->pStatePos;

×

3211

    SResultRow*        pRow = NULL;

×

3212

    SSessionKey*       pKey = (SSessionKey*)pPos->pKey;

×

3214

    if (pBlock->info.id.groupId == 0) {

×

3215

      pBlock->info.id.groupId = pKey->groupId;

×

3217

      void*   tbname = NULL;

×

3218

      int32_t winCode = TSDB_CODE_SUCCESS;

×

3219

      code = pAPI->stateStore.streamStateGetParName((void*)pTaskInfo->streamInfo.pState, pBlock->info.id.groupId,

×

3221

      QUERY_CHECK_CODE(code, lino, _end);

×

3223

      if (winCode != TSDB_CODE_SUCCESS) {

×

3224

        pBlock->info.parTbName[0] = 0;

×

3226

        memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);

×

3228

      pAPI->stateStore.streamStateFreeVal(tbname);

×

3231

      if (pBlock->info.id.groupId != pKey->groupId) {

×

3232

        break;

×

3236

    code = pAPI->stateStore.streamStateGetByPos(pState, pPos, (void**)&pRow);

×

3237

    QUERY_CHECK_CODE(code, lino, _end);

×

3239

    doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset);

×

3241

    if (pRow->numOfRows == 0) {

×

3242

      pGroupResInfo->index += 1;

×

3243

      continue;

×

3246

    if (pKey->win.ekey - pKey->win.skey < minWindowSize) {

×

3247

      qDebug("skip small window, groupId: %" PRId64 ", windowSize: %" PRId64 ", minWindowSize: %" PRId64, pKey->groupId,

×

3249

      pGroupResInfo->index += 1;

×

3250

      continue;

×

3253

    if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) {

×

3254

      break;

×

3257

    pGroupResInfo->index += 1;

×

3259

    for (int32_t j = 0; j < numOfExprs; ++j) {

×

3260

      int32_t slotId = pExprInfo[j].base.resSchema.slotId;

×

3262

      pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset);

×

3263

      if (pCtx[j].fpSet.finalize) {

×

3264

        int32_t tmpRes = pCtx[j].fpSet.finalize(&pCtx[j], pBlock);

×

3265

        if (TAOS_FAILED(tmpRes)) {

×

3266

          qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(tmpRes));

×

3267

          QUERY_CHECK_CODE(code, lino, _end);

×

3269

      } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) {

×

3274

        SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId);

×

3275

        char*            in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo);

×

3276

        for (int32_t k = 0; k < pRow->numOfRows; ++k) {

×

3277

          code = colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes);

×

3278

          QUERY_CHECK_CODE(code, lino, _end);

×

3283

    if (pSessionKeys) {

×

3284

      for (int32_t j = 0; j < pRow->numOfRows; ++j) {

×

3285

        const void* px = taosArrayPush(pSessionKeys, pKey);

×

3286

        QUERY_CHECK_NULL(px, code, lino, _end, terrno);

×

3290

    pBlock->info.dataLoad = 1;

×

3291

    pBlock->info.rows += pRow->numOfRows;

×

3293

  code = blockDataUpdateTsWindow(pBlock, 0);

×

3294

  QUERY_CHECK_CODE(code, lino, _end);

×

3296

_end:

×

3297

  if (code != TSDB_CODE_SUCCESS) {

×

3298

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3300

  return code;

×

3303

void doBuildSessionResult(SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo, SSDataBlock* pBlock,

×

3305

  int32_t        code = TSDB_CODE_SUCCESS;

×

3306

  int32_t        lino = 0;

×

3307

  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

×

3309

  pBlock->info.version = pTaskInfo->version;

×

3311

  blockDataCleanup(pBlock);

×

3312

  taosArrayClear(pSessionKeys);

×

3313

  if (!hasRemainResults(pGroupResInfo)) {

×

3314

    cleanupGroupResInfo(pGroupResInfo);

×

3315

    goto _end;

×

3319

  pBlock->info.id.groupId = 0;

×

3320

  code = buildSessionResultDataBlock(pOperator, pState, pBlock, &pOperator->exprSupp, pGroupResInfo, pSessionKeys);

×

3321

  QUERY_CHECK_CODE(code, lino, _end);

×

3323

  if (pBlock->info.rows == 0) {

×

3324

    cleanupGroupResInfo(pGroupResInfo);

×

3327

_end:

×

3328

  if (code != TSDB_CODE_SUCCESS) {

×

3329

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3331

×

3333

static int32_t buildSessionResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

3334

  int32_t                        code = TSDB_CODE_SUCCESS;

×

3335

  int32_t                        lino = 0;

×

3336

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3337

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

3338

  SOptrBasicInfo*                pBInfo = &pInfo->binfo;

×

3339

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

3340

  SStreamNotifyEventSupp*        pNotifySup = &pInfo->basic.notifyEventSup;

×

3341

  STaskNotifyEventStat*          pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;

×

3342

  bool                           addNotifyEvent = false;

×

3343

  addNotifyEvent = IS_NORMAL_SESSION_OP(pOperator) &&

×

3344

                   BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);

×

3345

  doBuildDeleteDataBlock(pOperator, pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator, &pInfo->groupResInfo);

×

3346

  if (pInfo->pDelRes->info.rows > 0) {

×

3347

    printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

3348

    if (addNotifyEvent) {

×

3349

      code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);

×

3350

      QUERY_CHECK_CODE(code, lino, _end);

×

3352

    (*ppRes) = pInfo->pDelRes;

×

3353

    return code;

×

3356

  doBuildSessionResult(pOperator, pAggSup->pState, &pInfo->groupResInfo, pBInfo->pRes,

×

3358

  if (pBInfo->pRes->info.rows > 0) {

×

3359

    printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

3360

    if (addNotifyEvent) {

×

3362

      for (int32_t i = 0; i < taosArrayGetSize(pNotifySup->pSessionKeys); ++i) {

×

3363

        SSessionKey* pKey = taosArrayGet(pNotifySup->pSessionKeys, i);

×

3364

        pKey->win.ekey += pAggSup->gap;

×

3366

      code = addAggResultNotifyEvent(pBInfo->pRes, pNotifySup->pSessionKeys, pTaskInfo->streamInfo.notifyResultSchema,

×

3368

      QUERY_CHECK_CODE(code, lino, _end);

×

3370

    (*ppRes) = pBInfo->pRes;

×

3371

    return code;

×

3374

  code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);

×

3375

  QUERY_CHECK_CODE(code, lino, _end);

×

3376

  if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {

×

3377

    printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

3378

    (*ppRes) = pNotifySup->pEventBlock;

×

3379

    return code;

×

3382

  code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);

×

3383

  QUERY_CHECK_CODE(code, lino, _end);

×

3385

_end:

×

3386

  if (code != TSDB_CODE_SUCCESS) {

×

3387

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3389

  (*ppRes) = NULL;

×

3390

  return code;

×

3393

int32_t getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins) {

×

3394

  int32_t code = TSDB_CODE_SUCCESS;

×

3395

  int32_t lino = 0;

×

3396

  int32_t size = taosArrayGetSize(pAllWins);

×

3397

  if (size == 0) {

×

3398

    goto _end;

×

3400

  SResultWindowInfo* pWinInfo = taosArrayGet(pAllWins, size - 1);

×

3401

  SSessionKey*       pSeKey = &pWinInfo->sessionWin;

×

3402

  void*              tmp = taosArrayPush(pMaxWins, pSeKey);

×

3403

  if (!tmp) {

×

3404

    code = terrno;

×

3405

    QUERY_CHECK_CODE(code, lino, _end);

×

3408

  if (pSeKey->groupId == 0) {

×

3409

    goto _end;

×

3411

  uint64_t preGpId = pSeKey->groupId;

×

3412

  for (int32_t i = size - 2; i >= 0; i--) {

×

3413

    pWinInfo = taosArrayGet(pAllWins, i);

×

3414

    pSeKey = &pWinInfo->sessionWin;

×

3415

    if (preGpId != pSeKey->groupId) {

×

3416

      void* tmp = taosArrayPush(pMaxWins, pSeKey);

×

3417

      if (!tmp) {

×

3418

        code = terrno;

×

3419

        QUERY_CHECK_CODE(code, lino, _end);

×

3421

      preGpId = pSeKey->groupId;

×

3425

_end:

×

3426

  if (code != TSDB_CODE_SUCCESS) {

×

3427

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3429

  return code;

×

3432

int32_t encodeSSessionKey(void** buf, SSessionKey* key) {

×

3433

  int32_t tlen = 0;

×

3434

  tlen += encodeSTimeWindow(buf, &key->win);

×

3435

  tlen += taosEncodeFixedU64(buf, key->groupId);

×

3436

  return tlen;

×

3439

void* decodeSSessionKey(void* buf, SSessionKey* key) {

×

3440

  buf = decodeSTimeWindow(buf, &key->win);

×

3441

  buf = taosDecodeFixedU64(buf, &key->groupId);

×

3442

  return buf;

×

3445

int32_t encodeSResultWindowInfo(void** buf, SResultWindowInfo* key, int32_t outLen) {

×

3446

  int32_t tlen = 0;

×

3447

  tlen += taosEncodeFixedBool(buf, key->isOutput);

×

3448

  tlen += encodeSSessionKey(buf, &key->sessionWin);

×

3449

  return tlen;

×

3452

void* decodeSResultWindowInfo(void* buf, SResultWindowInfo* key, int32_t outLen) {

×

3453

  buf = taosDecodeFixedBool(buf, &key->isOutput);

×

3454

  buf = decodeSSessionKey(buf, &key->sessionWin);

×

3455

  return buf;

×

3458

int32_t doStreamSessionEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOperator, bool isParent) {

×

3459

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3460

  if (!pInfo) {

×

3461

    return 0;

×

3464

  void* pData = (buf == NULL) ? NULL : *buf;

×

3467

  int32_t tlen = 0;

×

3468

  int32_t mapSize = tSimpleHashGetSize(pInfo->streamAggSup.pResultRows);

×

3469

  tlen += taosEncodeFixedI32(buf, mapSize);

×

3470

  void*   pIte = NULL;

×

3471

  size_t  keyLen = 0;

×

3472

  int32_t iter = 0;

×

3473

  while ((pIte = tSimpleHashIterate(pInfo->streamAggSup.pResultRows, pIte, &iter)) != NULL) {

×

3474

    void* key = tSimpleHashGetKey(pIte, &keyLen);

×

3475

    tlen += encodeSSessionKey(buf, key);

×

3476

    tlen += encodeSResultWindowInfo(buf, pIte, pInfo->streamAggSup.resultRowSize);

×

3480

  tlen += encodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);

×

3483

  int32_t size = taosArrayGetSize(pInfo->pChildren);

×

3484

  tlen += taosEncodeFixedI32(buf, size);

×

3485

  for (int32_t i = 0; i < size; i++) {

×

3486

    SOperatorInfo* pChOp = taosArrayGetP(pInfo->pChildren, i);

×

3487

    tlen += doStreamSessionEncodeOpState(buf, 0, pChOp, false);

×

3491

  tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);

×

3494

  tlen += encodeStreamBasicInfo(buf, &pInfo->basic);

×

3497

  if (isParent) {

×

3498

    if (buf) {

×

3499

      uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));

×

3500

      tlen += taosEncodeFixedU32(buf, cksum);

×

3502

      tlen += sizeof(uint32_t);

×

3506

  return tlen;

×

3509

int32_t doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator, bool isParent, void** ppBuf) {

×

3510

  int32_t                        code = TSDB_CODE_SUCCESS;

×

3511

  int32_t                        lino = 0;

×

3512

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3513

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

3514

  void*                        pDataEnd = POINTER_SHIFT(buf, len);

×

3515

  if (!pInfo) {

×

3516

    code = TSDB_CODE_FAILED;

×

3517

    QUERY_CHECK_CODE(code, lino, _end);

×

3519

  SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;

×

3522

  if (isParent) {

×

3523

    int32_t dataLen = len - sizeof(uint32_t);

×

3524

    void*   pCksum = POINTER_SHIFT(buf, dataLen);

×

3525

    if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) {

×

3526

      qError("stream session state is invalid");

×

3527

      code = TSDB_CODE_FAILED;

×

3528

      QUERY_CHECK_CODE(code, lino, _end);

×

3530

    pDataEnd = pCksum;

×

3534

  int32_t mapSize = 0;

×

3535

  buf = taosDecodeFixedI32(buf, &mapSize);

×

3536

  for (int32_t i = 0; i < mapSize; i++) {

×

3537

    SResultWindowInfo winfo = {0};

×

3538

    buf = decodeSSessionKey(buf, &winfo.sessionWin);

×

3539

    int32_t winCode = TSDB_CODE_SUCCESS;

×

3540

    code = pAggSup->stateStore.streamStateSessionAddIfNotExist(

×

3542

    QUERY_CHECK_CODE(code, lino, _end);

×

3543

    QUERY_CHECK_CONDITION((winCode == TSDB_CODE_SUCCESS), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);

×

3545

    buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize);

×

3546

    code = tSimpleHashPut(pInfo->streamAggSup.pResultRows, &winfo.sessionWin, sizeof(SSessionKey), &winfo,

×

3548

    QUERY_CHECK_CODE(code, lino, _end);

×

3552

  buf = decodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);

×

3555

  int32_t size = 0;

×

3556

  buf = taosDecodeFixedI32(buf, &size);

×

3557

  for (int32_t i = 0; i < size; i++) {

×

3558

    SOperatorInfo* pChOp = taosArrayGetP(pInfo->pChildren, i);

×

3559

    code = doStreamSessionDecodeOpState(buf, 0, pChOp, false, &buf);

×

3560

    QUERY_CHECK_CODE(code, lino, _end);

×

3564

  buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);

×

3565

  if (ppBuf) {

×

3566

    (*ppBuf) = buf;

×

3570

  if (buf < pDataEnd) {

×

3571

    code = decodeStreamBasicInfo(&buf, &pInfo->basic);

×

3572

    QUERY_CHECK_CODE(code, lino, _end);

×

3575

_end:

×

3576

  if (code != TSDB_CODE_SUCCESS) {

×

3577

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3579

  return code;

×

3582

void doStreamSessionSaveCheckpoint(SOperatorInfo* pOperator) {

×

3583

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3584

  if (needSaveStreamOperatorInfo(&pInfo->basic)) {

×

3585

    int32_t len = doStreamSessionEncodeOpState(NULL, 0, pOperator, true);

×

3586

    void*   buf = taosMemoryCalloc(1, len);

×

3587

    if (!buf) {

×

3588

      qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY));

×

3589

      return;

×

3591

    void* pBuf = buf;

×

3592

    len = doStreamSessionEncodeOpState(&pBuf, len, pOperator, true);

×

3593

    pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME,

×

3595

    taosMemoryFree(buf);

×

3596

    saveStreamOperatorStateComplete(&pInfo->basic);

×

3600

void resetUnCloseSessionWinInfo(SSHashObj* winMap) {

×

3601

  void*   pIte = NULL;

×

3602

  int32_t iter = 0;

×

3603

  while ((pIte = tSimpleHashIterate(winMap, pIte, &iter)) != NULL) {

×

3604

    SResultWindowInfo* pResInfo = pIte;

×

3605

    pResInfo->pStatePos->beUsed = true;

×

3607

×

3609

int32_t copyDeleteSessionKey(SSHashObj* source, SSHashObj* dest) {

×

3610

  int32_t code = TSDB_CODE_SUCCESS;

×

3611

  int32_t lino = 0;

×

3612

  if (tSimpleHashGetSize(source) == 0) {

×

3613

    goto _end;

×

3615

  void*   pIte = NULL;

×

3616

  int32_t iter = 0;

×

3617

  size_t  keyLen = 0;

×

3618

  while ((pIte = tSimpleHashIterate(source, pIte, &iter)) != NULL) {

×

3619

    SSessionKey* pKey = tSimpleHashGetKey(pIte, &keyLen);

×

3620

    code = saveDeleteRes(dest, *pKey);

×

3621

    QUERY_CHECK_CODE(code, lino, _end);

×

3623

  tSimpleHashClear(source);

×

3625

_end:

×

3626

  if (code != TSDB_CODE_SUCCESS) {

×

3627

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3629

  return code;

×

3632

static int32_t doStreamSessionAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

3633

  int32_t                        code = TSDB_CODE_SUCCESS;

×

3634

  int32_t                        lino = 0;

×

3635

  SExprSupp*                     pSup = &pOperator->exprSupp;

×

3636

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3637

  SOptrBasicInfo*                pBInfo = &pInfo->binfo;

×

3638

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

3639

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

3640

  qDebug("stask:%s  %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);

×

3641

  if (pOperator->status == OP_EXEC_DONE) {

×

3642

    (*ppRes) = NULL;

×

3643

    return code;

×

3644

  } else if (pOperator->status == OP_RES_TO_RETURN) {

×

3645

    SSDataBlock* opRes = NULL;

×

3646

    code = buildSessionResult(pOperator, &opRes);

×

3647

    QUERY_CHECK_CODE(code, lino, _end);

×

3648

    if (opRes) {

×

3649

      (*ppRes) = opRes;

×

3650

      return code;

×

3653

    if (pInfo->recvGetAll) {

×

3654

      pInfo->recvGetAll = false;

×

3655

      resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows);

×

3658

    if (pInfo->reCkBlock) {

×

3659

      pInfo->reCkBlock = false;

×

3660

      printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

3661

      (*ppRes) = pInfo->pCheckpointRes;

×

3662

      return code;

×

3665

    setStreamOperatorCompleted(pOperator);

×

3666

    (*ppRes) = NULL;

×

3667

    return code;

×

3670

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

3671

  if (!pInfo->pUpdated) {

×

3672

    pInfo->pUpdated = taosArrayInit(16, sizeof(SResultWindowInfo));

×

3673

    QUERY_CHECK_NULL(pInfo->pUpdated, code, lino, _end, terrno);

×

3675

  if (!pInfo->pStUpdated) {

×

3676

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

3677

    pInfo->pStUpdated = tSimpleHashInit(64, hashFn);

×

3678

    QUERY_CHECK_NULL(pInfo->pStUpdated, code, lino, _end, terrno);

×

3680

  while (1) {

×

3681

    SSDataBlock* pBlock = NULL;

×

3682

    code = downstream->fpSet.getNextFn(downstream, &pBlock);

×

3683

    QUERY_CHECK_CODE(code, lino, _end);

×

3685

    if (pBlock == NULL) {

×

3686

      break;

×

3688

    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));

×

3689

    setStreamOperatorState(&pInfo->basic, pBlock->info.type);

×

3691

    if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||

×

3692

        pBlock->info.type == STREAM_CLEAR) {

×

3693

      SArray* pWins = taosArrayInit(16, sizeof(SSessionKey));

×

3694

      QUERY_CHECK_NULL(pWins, code, lino, _end, terrno);

×

3696

      code = doDeleteTimeWindows(pAggSup, pBlock, pWins);

×

3697

      QUERY_CHECK_CODE(code, lino, _end);

×

3699

      removeSessionResults(pAggSup, pInfo->pStUpdated, pWins);

×

3700

      if (IS_FINAL_SESSION_OP(pOperator)) {

×

3701

        int32_t                        childIndex = getChildIndex(pBlock);

×

3702

        SOperatorInfo*                 pChildOp = taosArrayGetP(pInfo->pChildren, childIndex);

×

3703

        SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info;

×

3705

        code = doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, NULL);

×

3706

        QUERY_CHECK_CODE(code, lino, _end);

×

3708

        code = rebuildSessionWindow(pOperator, pWins, pInfo->pStUpdated);

×

3709

        QUERY_CHECK_CODE(code, lino, _end);

×

3711

      code = copyDeleteWindowInfo(pWins, pInfo->pStDeleted);

×

3712

      QUERY_CHECK_CODE(code, lino, _end);

×

3714

      if (pInfo->destHasPrimaryKey && IS_NORMAL_SESSION_OP(pOperator)) {

×

3715

        code = copyDeleteWindowInfo(pWins, pInfo->pPkDeleted);

×

3716

        QUERY_CHECK_CODE(code, lino, _end);

×

3718

      taosArrayDestroy(pWins);

×

3719

      continue;

×

3720

    } else if (pBlock->info.type == STREAM_GET_ALL) {

×

3721

      pInfo->recvGetAll = true;

×

3722

      code = getAllSessionWindow(pAggSup->pResultRows, pInfo->pStUpdated);

×

3723

      QUERY_CHECK_CODE(code, lino, _end);

×

3724

      continue;

×

3725

    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {

×

3726

      (*ppRes) = pBlock;

×

3727

      return code;

×

3728

    } else if (pBlock->info.type == STREAM_CHECKPOINT) {

×

3729

      pAggSup->stateStore.streamStateCommit(pAggSup->pState);

×

3730

      doStreamSessionSaveCheckpoint(pOperator);

×

3731

      code = copyDataBlock(pInfo->pCheckpointRes, pBlock);

×

3732

      QUERY_CHECK_CODE(code, lino, _end);

×

3734

      continue;

×

3736

      if (pBlock->info.type != STREAM_NORMAL && pBlock->info.type != STREAM_INVALID) {

×

3737

        code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;

×

3738

        QUERY_CHECK_CODE(code, lino, _end);

×

3742

    if (pInfo->scalarSupp.pExprInfo != NULL) {

×

3743

      SExprSupp* pExprSup = &pInfo->scalarSupp;

×

3744

      code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);

×

3745

      QUERY_CHECK_CODE(code, lino, _end);

×

3748

    code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

3749

    QUERY_CHECK_CODE(code, lino, _end);

×

3751

    doStreamSessionAggImpl(pOperator, pBlock, pInfo->pStUpdated, pInfo->pStDeleted, IS_FINAL_SESSION_OP(pOperator),

×

3753

    if (IS_FINAL_SESSION_OP(pOperator)) {

×

3754

      int32_t chIndex = getChildIndex(pBlock);

×

3755

      int32_t size = taosArrayGetSize(pInfo->pChildren);

×

3757

      for (int32_t i = 0; i < chIndex + 1 - size; i++) {

×

3758

        SOperatorInfo* pChildOp = NULL;

×

3759

        code = createStreamFinalSessionAggOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0, NULL, &pChildOp);

×

3760

        if (pChildOp == NULL || code != 0) {

×

3761

          qError("%s create stream child of final session error", GET_TASKID(pTaskInfo));

×

3762

          code = TSDB_CODE_FAILED;

×

3763

          QUERY_CHECK_CODE(code, lino, _end);

×

3766

        void* tmp = taosArrayPush(pInfo->pChildren, &pChildOp);

×

3767

        if (!tmp) {

×

3768

          code = terrno;

×

3769

          QUERY_CHECK_CODE(code, lino, _end);

×

3773

      SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex);

×

3774

      code = setInputDataBlock(&pChildOp->exprSupp, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

3775

      QUERY_CHECK_CODE(code, lino, _end);

×

3776

      doStreamSessionAggImpl(pChildOp, pBlock, NULL, NULL, true, false);

×

3778

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);

×

3779

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.watermark);

×

3782

  pOperator->status = OP_RES_TO_RETURN;

×

3784

  code = closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pInfo->pStUpdated);

×

3785

  QUERY_CHECK_CODE(code, lino, _end);

×

3787

  code = closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs);

×

3788

  QUERY_CHECK_CODE(code, lino, _end);

×

3790

  code = copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc);

×

3791

  QUERY_CHECK_CODE(code, lino, _end);

×

3793

  if (!pInfo->destHasPrimaryKey) {

×

3794

    removeSessionDeleteResults(pInfo->pStDeleted, pInfo->pUpdated);

×

3796

  if (pInfo->isHistoryOp) {

×

3797

    code = getMaxTsWins(pInfo->pUpdated, pInfo->historyWins);

×

3798

    QUERY_CHECK_CODE(code, lino, _end);

×

3800

  if (pInfo->destHasPrimaryKey && IS_NORMAL_SESSION_OP(pOperator)) {

×

3801

    code = copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pStDeleted);

×

3802

    QUERY_CHECK_CODE(code, lino, _end);

×

3804

  initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);

×

3805

  pInfo->pUpdated = NULL;

×

3806

  code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);

×

3807

  QUERY_CHECK_CODE(code, lino, _end);

×

3809

  SSDataBlock* opRes = NULL;

×

3810

  code = buildSessionResult(pOperator, &opRes);

×

3811

  QUERY_CHECK_CODE(code, lino, _end);

×

3812

  if (opRes) {

×

3813

    (*ppRes) = opRes;

×

3814

    return code;

×

3817

_end:

×

3818

  if (code != TSDB_CODE_SUCCESS) {

×

3819

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3820

    pTaskInfo->code = code;

×

3821

    T_LONG_JMP(pTaskInfo->env, code);

×

3823

  setStreamOperatorCompleted(pOperator);

×

3824

  (*ppRes) = NULL;

×

3825

  return code;

×

3828

static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {

×

3829

  SSDataBlock* pRes = NULL;

×

3830

  int32_t      code = doStreamSessionAggNext(pOperator, &pRes);

×

3831

  return pRes;

×

3834

void streamSessionReleaseState(SOperatorInfo* pOperator) {

×

3835

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3836

  int32_t                        winSize = taosArrayGetSize(pInfo->historyWins) * sizeof(SSessionKey);

×

3837

  int32_t                        resSize = winSize + sizeof(TSKEY);

×

3838

  char*                          pBuff = taosMemoryCalloc(1, resSize);

×

3839

  if (!pBuff) {

×

3840

    return;

×

3842

  memcpy(pBuff, pInfo->historyWins->pData, winSize);

×

3843

  memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY));

×

3844

  pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_STATE_NAME,

×

3846

  pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState);

×

3847

  taosMemoryFreeClear(pBuff);

×

3848

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

3849

  if (downstream->fpSet.releaseStreamStateFn) {

×

3850

    downstream->fpSet.releaseStreamStateFn(downstream);

×

3854

void resetWinRange(STimeWindow* winRange) {

×

3855

  winRange->skey = INT64_MIN;

×

3856

  winRange->ekey = INT64_MAX;

×

3857

×

3859

int32_t getSessionWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SResultWindowInfo* pWinInfo) {

×

3860

  int32_t code = TSDB_CODE_SUCCESS;

×

3861

  int32_t lino = 0;

×

3862

  int32_t rowSize = pAggSup->resultRowSize;

×

3863

  int32_t winCode = TSDB_CODE_SUCCESS;

×

3864

  code = pAggSup->stateStore.streamStateSessionGet(pAggSup->pState, pKey, (void**)&pWinInfo->pStatePos, &rowSize,

×

3866

  QUERY_CHECK_CODE(code, lino, _end);

×

3868

  if (winCode == TSDB_CODE_SUCCESS) {

×

3869

    pWinInfo->sessionWin = *pKey;

×

3870

    pWinInfo->isOutput = true;

×

3871

    if (pWinInfo->pStatePos->needFree) {

×

3872

      pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pWinInfo->sessionWin);

×

3875

    SET_SESSION_WIN_INVALID((*pWinInfo));

×

3878

_end:

×

3879

  if (code != TSDB_CODE_SUCCESS) {

×

3880

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

3882

  return code;

×

3885

void reloadAggSupFromDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup) {

×

3886

  SStateStore* pAPI = &downstream->pTaskInfo->storageAPI.stateStore;

×

3888

  if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {

×

3889

    reloadAggSupFromDownStream(downstream->pDownstream[0], pAggSup);

×

3890

    return;

×

3893

  SStreamScanInfo* pScanInfo = downstream->info;

×

3894

  pAggSup->pUpdateInfo = pScanInfo->pUpdateInfo;

×

3897

void streamSessionSemiReloadState(SOperatorInfo* pOperator) {

×

3898

  int32_t                        code = TSDB_CODE_SUCCESS;

×

3899

  int32_t                        lino = 0;

×

3900

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3901

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

3902

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

3903

  resetWinRange(&pAggSup->winRange);

×

3905

  SResultWindowInfo winInfo = {0};

×

3906

  int32_t           size = 0;

×

3907

  void*             pBuf = NULL;

×

3908

  code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME,

×

3910

  QUERY_CHECK_CODE(code, lino, _end);

×

3912

  int32_t      num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);

×

3913

  SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;

×

3914

  for (int32_t i = 0; i < num; i++) {

×

3915

    SResultWindowInfo winInfo = {0};

×

3916

    code = getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &winInfo);

×

3917

    QUERY_CHECK_CODE(code, lino, _end);

×

3918

    if (!IS_VALID_SESSION_WIN(winInfo)) {

×

3919

      continue;

×

3921

    compactSessionSemiWindow(pOperator, &winInfo);

×

3922

    code = saveSessionOutputBuf(pAggSup, &winInfo);

×

3923

    QUERY_CHECK_CODE(code, lino, _end);

×

3925

  TSKEY ts = *(TSKEY*)((char*)pBuf + size - sizeof(TSKEY));

×

3926

  taosMemoryFree(pBuf);

×

3927

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);

×

3928

  pAggSup->stateStore.streamStateReloadInfo(pAggSup->pState, ts);

×

3930

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

3931

  if (downstream->fpSet.reloadStreamStateFn) {

×

3932

    downstream->fpSet.reloadStreamStateFn(downstream);

×

3934

  reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup);

×

3936

_end:

×

3937

  if (code != TSDB_CODE_SUCCESS) {

×

3938

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

3940

×

3942

void streamSessionReloadState(SOperatorInfo* pOperator) {

×

3943

  int32_t                        code = TSDB_CODE_SUCCESS;

×

3944

  int32_t                        lino = 0;

×

3945

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

3946

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

3947

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

3948

  resetWinRange(&pAggSup->winRange);

×

3950

  int32_t size = 0;

×

3951

  void*   pBuf = NULL;

×

3952

  code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_SESSION_OP_STATE_NAME,

×

3955

  QUERY_CHECK_CODE(code, lino, _end);

×

3957

  int32_t      num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);

×

3958

  SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;

×

3960

  TSKEY ts = *(TSKEY*)((char*)pBuf + size - sizeof(TSKEY));

×

3961

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);

×

3962

  pAggSup->stateStore.streamStateReloadInfo(pAggSup->pState, ts);

×

3964

  if (!pInfo->pStUpdated && num > 0) {

×

3965

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

3966

    pInfo->pStUpdated = tSimpleHashInit(64, hashFn);

×

3967

    QUERY_CHECK_NULL(pInfo->pStUpdated, code, lino, _end, terrno);

×

3969

  for (int32_t i = 0; i < num; i++) {

×

3970

    SResultWindowInfo winInfo = {0};

×

3971

    code = getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &winInfo);

×

3972

    QUERY_CHECK_CODE(code, lino, _end);

×

3973

    if (!IS_VALID_SESSION_WIN(winInfo)) {

×

3974

      continue;

×

3977

    int32_t winNum = 0;

×

3978

    code = compactSessionWindow(pOperator, &winInfo, pInfo->pStUpdated, pInfo->pStDeleted, true, &winNum);

×

3979

    QUERY_CHECK_CODE(code, lino, _end);

×

3981

    if (winNum > 0) {

×

3982

      qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, winInfo.sessionWin.win.skey,

×

3984

      if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {

×

3985

        code = saveResult(winInfo, pInfo->pStUpdated);

×

3986

        QUERY_CHECK_CODE(code, lino, _end);

×

3987

      } else if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

3988

        if (!isCloseWindow(&winInfo.sessionWin.win, &pInfo->twAggSup)) {

×

3989

          code = saveDeleteRes(pInfo->pStDeleted, winInfo.sessionWin);

×

3990

          QUERY_CHECK_CODE(code, lino, _end);

×

3992

        SSessionKey key = {0};

×

3993

        getSessionHashKey(&winInfo.sessionWin, &key);

×

3994

        code = tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo));

×

3995

        QUERY_CHECK_CODE(code, lino, _end);

×

3998

    code = saveSessionOutputBuf(pAggSup, &winInfo);

×

3999

    QUERY_CHECK_CODE(code, lino, _end);

×

4001

  taosMemoryFree(pBuf);

×

4003

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

4004

  if (downstream->fpSet.reloadStreamStateFn) {

×

4005

    downstream->fpSet.reloadStreamStateFn(downstream);

×

4007

  reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup);

×

4009

_end:

×

4010

  if (code != TSDB_CODE_SUCCESS) {

×

4011

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

4013

×

4015

int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,

×

4017

  QRY_PARAM_CHECK(pOptrInfo);

×

4019

  SSessionWinodwPhysiNode*       pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode;

×

4020

  int32_t                        numOfCols = 0;

×

4021

  int32_t                        code = TSDB_CODE_OUT_OF_MEMORY;

×

4022

  int32_t                        lino = 0;

×

4023

  SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo));

×

4024

  SOperatorInfo*                 pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));

×

4025

  if (pInfo == NULL || pOperator == NULL) {

×

4026

    code = terrno;

×

4027

    goto _error;

×

4030

  pOperator->pTaskInfo = pTaskInfo;

×

4032

  initResultSizeInfo(&pOperator->resultInfo, 4096);

×

4033

  if (pSessionNode->window.pExprs != NULL) {

×

4034

    int32_t    numOfScalar = 0;

×

4035

    SExprInfo* pScalarExprInfo = NULL;

×

4036

    code = createExprInfo(pSessionNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalar);

×

4037

    QUERY_CHECK_CODE(code, lino, _error);

×

4039

    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);

×

4040

    if (code != TSDB_CODE_SUCCESS) {

×

4041

      goto _error;

×

4044

  SExprSupp* pExpSup = &pOperator->exprSupp;

×

4046

  SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);

×

4047

  QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);

×

4048

  pInfo->binfo.pRes = pResBlock;

×

4050

  SExprInfo* pExprInfo = NULL;

×

4051

  code = createExprInfo(pSessionNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);

×

4052

  QUERY_CHECK_CODE(code, lino, _error);

×

4054

  code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore);

×

4055

  QUERY_CHECK_CODE(code, lino, _error);

×

4057

  pInfo->twAggSup = (STimeWindowAggSupp){

×

4058

      .waterMark = pSessionNode->window.watermark,

×

4059

      .calTrigger = pSessionNode->window.triggerType,

×

4062

      .deleteMark = getDeleteMark(&pSessionNode->window, 0),

×

4065

  pInfo->primaryTsIndex = ((SColumnNode*)pSessionNode->window.pTspk)->slotId;

×

4067

      initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, pSessionNode->gap, pTaskInfo->streamInfo.pState,

×

4068

                             0, 0, &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup, GET_TASKID(pTaskInfo),

×

4070

  if (code != TSDB_CODE_SUCCESS) {

×

4071

    goto _error;

×

4074

  code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

×

4075

  QUERY_CHECK_CODE(code, lino, _error);

×

4077

  if (pSessionNode->window.pTsEnd) {

×

4078

    pInfo->endTsIndex = ((SColumnNode*)pSessionNode->window.pTsEnd)->slotId;

×

4081

  pInfo->order = TSDB_ORDER_ASC;

×

4082

  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

4083

  pInfo->pStDeleted = tSimpleHashInit(64, hashFn);

×

4084

  QUERY_CHECK_NULL(pInfo->pStDeleted, code, lino, _error, terrno);

×

4085

  pInfo->pDelIterator = NULL;

×

4086

  code = createSpecialDataBlock(STREAM_DELETE_RESULT, &pInfo->pDelRes);

×

4087

  QUERY_CHECK_CODE(code, lino, _error);

×

4089

  pInfo->pChildren = NULL;

×

4090

  pInfo->pPhyNode = pPhyNode;

×

4091

  pInfo->ignoreExpiredData = pSessionNode->window.igExpired;

×

4092

  pInfo->ignoreExpiredDataSaved = false;

×

4093

  pInfo->pUpdated = NULL;

×

4094

  pInfo->pStUpdated = NULL;

×

4095

  pInfo->dataVersion = 0;

×

4096

  pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey));

×

4097

  if (!pInfo->historyWins) {

×

4098

    goto _error;

×

4100

  if (pHandle) {

×

4101

    pInfo->isHistoryOp = pHandle->fillHistory;

×

4104

  code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);

×

4105

  QUERY_CHECK_CODE(code, lino, _error);

×

4107

  pInfo->clearState = false;

×

4108

  pInfo->recvGetAll = false;

×

4109

  pInfo->destHasPrimaryKey = pSessionNode->window.destHasPrimaryKey;

×

4110

  pInfo->pPkDeleted = tSimpleHashInit(64, hashFn);

×

4111

  QUERY_CHECK_NULL(pInfo->pPkDeleted, code, lino, _error, terrno);

×

4112

  pInfo->pOperator = pOperator;

×

4114

  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;

×

4115

  setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true,

×

4117

  if (pPhyNode->type != QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {

×

4119

    void*   buff = NULL;

×

4120

    int32_t len = 0;

×

4122

        pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME,

×

4124

    if (res == TSDB_CODE_SUCCESS) {

×

4125

      code = doStreamSessionDecodeOpState(buff, len, pOperator, true, NULL);

×

4126

      taosMemoryFree(buff);

×

4127

      QUERY_CHECK_CODE(code, lino, _error);

×

4131

      createOperatorFpSet(optrDummyOpenFn, doStreamSessionAggNext, NULL, destroyStreamSessionAggOperatorInfo,

×

4133

  setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionReloadState);

×

4135

  code = initStreamBasicInfo(&pInfo->basic, pOperator);

×

4136

  QUERY_CHECK_CODE(code, lino, _error);

×

4138

  if (downstream) {

×

4139

    pInfo->basic.primaryPkIndex = -1;

×

4140

    code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,

×

4141

                          &pInfo->twAggSup, &pInfo->basic);

×

4142

    QUERY_CHECK_CODE(code, lino, _error);

×

4144

    code = appendDownstream(pOperator, &downstream, 1);

×

4145

    QUERY_CHECK_CODE(code, lino, _error);

×

4148

  *pOptrInfo = pOperator;

×

4149

  return TSDB_CODE_SUCCESS;

×

4151

_error:

×

4152

  if (pInfo != NULL) {

×

4153

    destroyStreamSessionAggOperatorInfo(pInfo);

×

4155

  destroyOperatorAndDownstreams(pOperator, &downstream, 1);

×

4156

  pTaskInfo->code = code;

×

4157

  qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

4158

  return code;

×

4161

static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) {

×

4162

  tSimpleHashClear(pInfo->streamAggSup.pResultRows);

×

4163

  pInfo->streamAggSup.stateStore.streamStateSessionClear(pInfo->streamAggSup.pState);

×

4164

  pInfo->clearState = false;

×

4165

×

4167

int32_t deleteSessionWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate,

×

4169

  int32_t code = TSDB_CODE_SUCCESS;

×

4170

  int32_t lino = 0;

×

4171

  SArray* pWins = taosArrayInit(16, sizeof(SSessionKey));

×

4172

  if (!pWins) {

×

4173

    code = terrno;

×

4174

    QUERY_CHECK_CODE(code, lino, _end);

×

4176

  code = doDeleteTimeWindows(pAggSup, pBlock, pWins);

×

4177

  QUERY_CHECK_CODE(code, lino, _end);

×

4179

  removeSessionResults(pAggSup, pMapUpdate, pWins);

×

4180

  code = copyDeleteWindowInfo(pWins, pMapDelete);

×

4181

  QUERY_CHECK_CODE(code, lino, _end);

×

4183

  if (needAdd) {

×

4184

    code = copyDeleteWindowInfo(pWins, pPkDelete);

×

4185

    QUERY_CHECK_CODE(code, lino, _end);

×

4187

  taosArrayDestroy(pWins);

×

4189

_end:

×

4190

  if (code != TSDB_CODE_SUCCESS) {

×

4191

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

4193

  return code;

×

4196

static int32_t doStreamSessionSemiAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

4197

  int32_t                        code = TSDB_CODE_SUCCESS;

×

4198

  int32_t                        lino = 0;

×

4199

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

4200

  SOptrBasicInfo*                pBInfo = &pInfo->binfo;

×

4201

  TSKEY                          maxTs = INT64_MIN;

×

4202

  SExprSupp*                     pSup = &pOperator->exprSupp;

×

4203

  SStreamAggSupporter*           pAggSup = &pInfo->streamAggSup;

×

4204

  SExecTaskInfo*                 pTaskInfo = pOperator->pTaskInfo;

×

4206

  qDebug("stask:%s  %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);

×

4207

  if (pOperator->status == OP_EXEC_DONE) {

×

4208

    (*ppRes) = NULL;

×

4209

    return code;

×

4213

    SSDataBlock* opRes = NULL;

×

4214

    code = buildSessionResult(pOperator, &opRes);

×

4215

    QUERY_CHECK_CODE(code, lino, _end);

×

4216

    if (opRes) {

×

4217

      (*ppRes) = opRes;

×

4218

      return code;

×

4221

    if (pInfo->clearState) {

×

4222

      clearFunctionContext(&pOperator->exprSupp);

×

4224

      clearStreamSessionOperator(pInfo);

×

4227

    if (pOperator->status == OP_RES_TO_RETURN) {

×

4228

      if (pInfo->reCkBlock) {

×

4229

        pInfo->reCkBlock = false;

×

4230

        printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

4231

        (*ppRes) = pInfo->pCheckpointRes;

×

4232

        return code;

×

4234

      clearFunctionContext(&pOperator->exprSupp);

×

4236

      clearStreamSessionOperator(pInfo);

×

4237

      setStreamOperatorCompleted(pOperator);

×

4238

      (*ppRes) = NULL;

×

4239

      return code;

×

4243

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

4244

  if (!pInfo->pUpdated) {

×

4245

    pInfo->pUpdated = taosArrayInit(16, sizeof(SResultWindowInfo));

×

4246

    QUERY_CHECK_NULL(pInfo->pUpdated, code, lino, _end, terrno);

×

4248

  if (!pInfo->pStUpdated) {

×

4249

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

4250

    pInfo->pStUpdated = tSimpleHashInit(64, hashFn);

×

4251

    QUERY_CHECK_NULL(pInfo->pStUpdated, code, lino, _end, terrno);

×

4253

  while (1) {

×

4254

    SSDataBlock* pBlock = NULL;

×

4255

    code = downstream->fpSet.getNextFn(downstream, &pBlock);

×

4256

    QUERY_CHECK_CODE(code, lino, _end);

×

4258

    if (pBlock == NULL) {

×

4259

      pOperator->status = OP_RES_TO_RETURN;

×

4260

      break;

×

4262

    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));

×

4263

    setStreamOperatorState(&pInfo->basic, pBlock->info.type);

×

4265

    if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||

×

4266

        pBlock->info.type == STREAM_CLEAR) {

×

4268

      code = deleteSessionWinState(pAggSup, pBlock, pInfo->pStUpdated, pInfo->pStDeleted, NULL, false);

×

4269

      QUERY_CHECK_CODE(code, lino, _end);

×

4270

      pInfo->clearState = true;

×

4271

      break;

×

4272

    } else if (pBlock->info.type == STREAM_GET_ALL) {

×

4273

      code = getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pStUpdated);

×

4274

      QUERY_CHECK_CODE(code, lino, _end);

×

4275

      continue;

×

4276

    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {

×

4277

      (*ppRes) = pBlock;

×

4278

      return code;

×

4279

    } else if (pBlock->info.type == STREAM_CHECKPOINT) {

×

4280

      pAggSup->stateStore.streamStateCommit(pAggSup->pState);

×

4281

      doStreamSessionSaveCheckpoint(pOperator);

×

4282

      continue;

×

4284

      if (pBlock->info.type != STREAM_NORMAL && pBlock->info.type != STREAM_INVALID) {

×

4285

        code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;

×

4286

        QUERY_CHECK_CODE(code, lino, _end);

×

4290

    if (pInfo->scalarSupp.pExprInfo != NULL) {

×

4291

      SExprSupp* pExprSup = &pInfo->scalarSupp;

×

4292

      code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);

×

4293

      QUERY_CHECK_CODE(code, lino, _end);

×

4296

    code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

4297

    QUERY_CHECK_CODE(code, lino, _end);

×

4298

    doStreamSessionAggImpl(pOperator, pBlock, pInfo->pStUpdated, NULL, false, false);

×

4299

    maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);

×

4302

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);

×

4303

  pBInfo->pRes->info.watermark = pInfo->twAggSup.maxTs;

×

4305

  code = copyUpdateResult(&pInfo->pStUpdated, pInfo->pUpdated, sessionKeyCompareAsc);

×

4306

  QUERY_CHECK_CODE(code, lino, _end);

×

4308

  removeSessionDeleteResults(pInfo->pStDeleted, pInfo->pUpdated);

×

4310

  if (pInfo->isHistoryOp) {

×

4311

    code = getMaxTsWins(pInfo->pUpdated, pInfo->historyWins);

×

4312

    QUERY_CHECK_CODE(code, lino, _end);

×

4315

  initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);

×

4316

  pInfo->pUpdated = NULL;

×

4317

  code = blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);

×

4318

  QUERY_CHECK_CODE(code, lino, _end);

×

4320

  SSDataBlock* opRes = NULL;

×

4321

  code = buildSessionResult(pOperator, &opRes);

×

4322

  QUERY_CHECK_CODE(code, lino, _end);

×

4323

  if (opRes) {

×

4324

    (*ppRes) = opRes;

×

4325

    return code;

×

4328

_end:

×

4329

  if (code != TSDB_CODE_SUCCESS) {

×

4330

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

4331

    pTaskInfo->code = code;

×

4332

    T_LONG_JMP(pTaskInfo->env, code);

×

4335

  clearFunctionContext(&pOperator->exprSupp);

×

4337

  clearStreamSessionOperator(pInfo);

×

4338

  setStreamOperatorCompleted(pOperator);

×

4339

  (*ppRes) = NULL;

×

4340

  return code;

×

4343

static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) {

×

4344

  SSDataBlock* pRes = NULL;

×

4345

  int32_t      code = doStreamSessionSemiAggNext(pOperator, &pRes);

×

4346

  return pRes;

×

4349

int32_t createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,

×

4352

  QRY_PARAM_CHECK(pOptrInfo);

×

4354

  int32_t        code = TSDB_CODE_SUCCESS;

×

4355

  int32_t        lino = 0;

×

4356

  SOperatorInfo* pOperator = NULL;

×

4357

  code = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo, pHandle, &pOperator);

×

4358

  if (pOperator == NULL || code != 0) {

×

4359

    downstream = NULL;

×

4360

    QUERY_CHECK_CODE(code, lino, _error);

×

4363

  SStorageAPI*                   pAPI = &pTaskInfo->storageAPI;

×

4364

  SStreamSessionAggOperatorInfo* pInfo = pOperator->info;

×

4365

  pOperator->operatorType = pPhyNode->type;

×

4366

  pInfo->pOperator = pOperator;

×

4368

  if (pPhyNode->type != QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) {

×

4369

    pOperator->fpSet =

×

4370

        createOperatorFpSet(optrDummyOpenFn, doStreamSessionSemiAggNext, NULL, destroyStreamSessionAggOperatorInfo,

×

4372

    setOperatorStreamStateFn(pOperator, streamSessionReleaseState, streamSessionSemiReloadState);

×

4375

  code = initStreamBasicInfo(&pInfo->basic, pOperator);

×

4376

  QUERY_CHECK_CODE(code, lino, _error);

×

4378

  setOperatorInfo(pOperator, getStreamOpName(pOperator->operatorType), pPhyNode->type, false, OP_NOT_OPENED, pInfo,

×

4381

  if (numOfChild > 0) {

×

4382

    pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*));

×

4383

    QUERY_CHECK_NULL(pInfo->pChildren, code, lino, _error, terrno);

×

4384

    for (int32_t i = 0; i < numOfChild; i++) {

×

4385

      SOperatorInfo* pChildOp = NULL;

×

4386

      code = createStreamFinalSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo, 0, pHandle, &pChildOp);

×

4387

      if (pChildOp == NULL || code != 0) {

×

4388

        QUERY_CHECK_CODE(code, lino, _error);

×

4391

      SStreamSessionAggOperatorInfo* pChInfo = pChildOp->info;

×

4392

      pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;

×

4393

      pAPI->stateStore.streamStateSetNumber(pChInfo->streamAggSup.pState, i, pInfo->primaryTsIndex);

×

4394

      void* tmp = taosArrayPush(pInfo->pChildren, &pChildOp);

×

4395

      if (!tmp) {

×

4396

        code = terrno;

×

4397

        QUERY_CHECK_CODE(code, lino, _error);

×

4401

    void*   buff = NULL;

×

4402

    int32_t len = 0;

×

4404

        pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_SESSION_OP_CHECKPOINT_NAME,

×

4406

    if (res == TSDB_CODE_SUCCESS) {

×

4407

      code = doStreamSessionDecodeOpState(buff, len, pOperator, true, NULL);

×

4408

      taosMemoryFree(buff);

×

4409

      QUERY_CHECK_CODE(code, lino, _error);

×

4413

  if (!IS_FINAL_SESSION_OP(pOperator) || numOfChild == 0) {

×

4414

    pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE;

×

4417

  *pOptrInfo = pOperator;

×

4418

  return code;

×

4420

_error:

×

4421

  if (pInfo != NULL) {

×

4422

    destroyStreamSessionAggOperatorInfo(pInfo);

×

4424

  if (pOperator != NULL) {

×

4425

    pOperator->info = NULL;

×

4426

    destroyOperator(pOperator);

×

4428

  pTaskInfo->code = code;

×

4429

  if (code != TSDB_CODE_SUCCESS) {

×

4430

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

4432

  return code;

×

4435

void destroyStreamStateOperatorInfo(void* param) {

×

4436

  if (param == NULL) {

×

4437

    return;

×

4439

  SStreamStateAggOperatorInfo* pInfo = (SStreamStateAggOperatorInfo*)param;

×

4440

  cleanupBasicInfo(&pInfo->binfo);

×

4441

  if (pInfo->pOperator) {

×

4442

    cleanupResultInfoInStream(pInfo->pOperator->pTaskInfo, pInfo->streamAggSup.pState, &pInfo->pOperator->exprSupp,

×

4444

    pInfo->pOperator = NULL;

×

4447

  destroyStreamBasicInfo(&pInfo->basic);

×

4449

  clearGroupResInfo(&pInfo->groupResInfo);

×

4450

  taosArrayDestroyP(pInfo->pUpdated, destroyFlusedPos);

×

4451

  pInfo->pUpdated = NULL;

×

4452

  destroyStreamAggSupporter(&pInfo->streamAggSup);

×

4454

  cleanupExprSupp(&pInfo->scalarSupp);

×

4455

  if (pInfo->pChildren != NULL) {

×

4456

    int32_t size = taosArrayGetSize(pInfo->pChildren);

×

4457

    for (int32_t i = 0; i < size; i++) {

×

4458

      SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, i);

×

4459

      destroyOperator(pChild);

×

4461

    taosArrayDestroy(pInfo->pChildren);

×

4463

  colDataDestroy(&pInfo->twAggSup.timeWindowData);

×

4464

  blockDataDestroy(pInfo->pDelRes);

×

4465

  tSimpleHashCleanup(pInfo->pSeUpdated);

×

4466

  tSimpleHashCleanup(pInfo->pSeDeleted);

×

4467

  cleanupGroupResInfo(&pInfo->groupResInfo);

×

4469

  taosArrayDestroy(pInfo->historyWins);

×

4470

  blockDataDestroy(pInfo->pCheckpointRes);

×

4471

  tSimpleHashCleanup(pInfo->pPkDeleted);

×

4473

  taosMemoryFreeClear(param);

×

4476

bool isTsInWindow(SStateWindowInfo* pWin, TSKEY ts) {

×

4477

  if (pWin->winInfo.sessionWin.win.skey <= ts && ts <= pWin->winInfo.sessionWin.win.ekey) {

×

4478

    return true;

×

4480

  return false;

×

4483

bool isEqualStateKey(SStateWindowInfo* pWin, char* pKeyData) {

×

4484

  return pKeyData && compareVal(pKeyData, pWin->pStateKey);

×

4487

bool compareStateKey(void* data, void* key) {

×

4488

  if (!data || !key) {

×

4489

    return true;

×

4491

  SStateKeys* stateKey = (SStateKeys*)key;

×

4492

  stateKey->pData = (char*)key + sizeof(SStateKeys);

×

4493

  return compareVal(data, stateKey);

×

4496

bool compareWinStateKey(SStateKeys* left, SStateKeys* right) {

×

4497

  if (!left || !right) {

×

4498

    return false;

×

4500

  return compareVal(left->pData, right);

×

4503

static void getNextStateWin(const SStreamAggSupporter* pAggSup, SStateWindowInfo* pNextWin, bool asc) {

×

4504

  SStreamStateCur* pCur = NULL;

×

4506

  if (pAggSup == NULL || pNextWin == NULL) {

×

4507

    return;

×

4510

  if (asc)

×

4511

    pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pNextWin->winInfo.sessionWin);

×

4513

    pCur = pAggSup->stateStore.streamStateSessionSeekKeyPrev(pAggSup->pState, &pNextWin->winInfo.sessionWin);

×

4514

  int32_t nextSize = pAggSup->resultRowSize;

×

4515

  int32_t winCode = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &pNextWin->winInfo.sessionWin,

×

4516

                                                                     (void**)&pNextWin->winInfo.pStatePos, &nextSize);

×

4517

  if (winCode != TSDB_CODE_SUCCESS) {

×

4518

    SET_SESSION_WIN_INVALID(pNextWin->winInfo);

×

4520

    pNextWin->pStateKey =

×

4521

        (SStateKeys*)((char*)pNextWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));

×

4522

    pNextWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);

×

4523

    pNextWin->pStateKey->type = pAggSup->stateKeyType;

×

4524

    pNextWin->pStateKey->pData = (char*)pNextWin->pStateKey + sizeof(SStateKeys);

×

4525

    pNextWin->pStateKey->isNull = false;

×

4526

    pNextWin->winInfo.isOutput = true;

×

4528

  pAggSup->stateStore.streamStateFreeCur(pCur);

×

4531

int32_t getStateWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SStateWindowInfo* pCurWin,

×

4533

  int32_t code = TSDB_CODE_SUCCESS;

×

4534

  int32_t lino = 0;

×

4535

  int32_t size = pAggSup->resultRowSize;

×

4536

  pCurWin->winInfo.sessionWin.groupId = pKey->groupId;

×

4537

  pCurWin->winInfo.sessionWin.win.skey = pKey->win.skey;

×

4538

  pCurWin->winInfo.sessionWin.win.ekey = pKey->win.ekey;

×

4539

  code = getSessionWindowInfoByKey(pAggSup, pKey, &pCurWin->winInfo);

×

4540

  QUERY_CHECK_CODE(code, lino, _end);

×

4541

  QUERY_CHECK_CONDITION((IS_VALID_SESSION_WIN(pCurWin->winInfo)), code, lino, _end,

×

4544

  pCurWin->pStateKey =

×

4545

      (SStateKeys*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));

×

4546

  pCurWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);

×

4547

  pCurWin->pStateKey->type = pAggSup->stateKeyType;

×

4548

  pCurWin->pStateKey->pData = (char*)pCurWin->pStateKey + sizeof(SStateKeys);

×

4549

  pCurWin->pStateKey->isNull = false;

×

4550

  pCurWin->winInfo.isOutput = true;

×

4551

  if (pCurWin->winInfo.pStatePos->needFree) {

×

4552

    pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->winInfo.sessionWin);

×

4555

  qDebug("===stream===get state cur win buff. skey:%" PRId64 ", endkey:%" PRId64, pCurWin->winInfo.sessionWin.win.skey,

×

4558

  pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin;

×

4559

  getNextStateWin(pAggSup, pNextWin, true);

×

4561

_end:

×

4562

  qDebug("===stream===get state next win buff. skey:%" PRId64 ", endkey:%" PRId64,

×

4564

  if (code != TSDB_CODE_SUCCESS) {

×

4565

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

4567

  return code;

×

4570

int32_t setStateOutputBuf(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData,

×

4572

  int32_t size = pAggSup->resultRowSize;

×

4573

  pCurWin->winInfo.sessionWin.groupId = groupId;

×

4574

  pCurWin->winInfo.sessionWin.win.skey = ts;

×

4575

  pCurWin->winInfo.sessionWin.win.ekey = ts;

×

4576

  int32_t code = TSDB_CODE_SUCCESS;

×

4577

  int32_t lino = 0;

×

4578

  int32_t winCode = TSDB_CODE_SUCCESS;

×

4579

  code = pAggSup->stateStore.streamStateStateAddIfNotExist(pAggSup->pState, &pCurWin->winInfo.sessionWin, pKeyData,

×

4581

                                                           (void**)&pCurWin->winInfo.pStatePos, &size, &winCode);

×

4582

  QUERY_CHECK_CODE(code, lino, _end);

×

4584

  pCurWin->pStateKey =

×

4585

      (SStateKeys*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));

×

4586

  pCurWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);

×

4587

  pCurWin->pStateKey->type = pAggSup->stateKeyType;

×

4588

  pCurWin->pStateKey->pData = (char*)pCurWin->pStateKey + sizeof(SStateKeys);

×

4589

  pCurWin->pStateKey->isNull = false;

×

4591

  if (winCode == TSDB_CODE_SUCCESS && !inWinRange(&pAggSup->winRange, &pCurWin->winInfo.sessionWin.win)) {

×

4592

    winCode = TSDB_CODE_FAILED;

×

4593

    clearOutputBuf(pAggSup->pState, pCurWin->winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);

×

4594

    pCurWin->pStateKey =

×

4595

        (SStateKeys*)((char*)pCurWin->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize));

×

4596

    pCurWin->pStateKey->bytes = pAggSup->stateKeySize - sizeof(SStateKeys);

×

4597

    pCurWin->pStateKey->type = pAggSup->stateKeyType;

×

4598

    pCurWin->pStateKey->pData = (char*)pCurWin->pStateKey + sizeof(SStateKeys);

×

4599

    pCurWin->pStateKey->isNull = false;

×

4600

    pCurWin->winInfo.sessionWin.groupId = groupId;

×

4601

    pCurWin->winInfo.sessionWin.win.skey = ts;

×

4602

    pCurWin->winInfo.sessionWin.win.ekey = ts;

×

4603

    qDebug("===stream===reset state win key. skey:%" PRId64 ", endkey:%" PRId64, pCurWin->winInfo.sessionWin.win.skey,

×

4607

  if (winCode == TSDB_CODE_SUCCESS) {

×

4608

    pCurWin->winInfo.isOutput = true;

×

4609

    if (pCurWin->winInfo.pStatePos->needFree) {

×

4610

      pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->winInfo.sessionWin);

×

4612

  } else if (pKeyData) {

×

4613

    if (IS_VAR_DATA_TYPE(pAggSup->stateKeyType)) {

×

4614

      varDataCopy(pCurWin->pStateKey->pData, pKeyData);

×

4616

      memcpy(pCurWin->pStateKey->pData, pKeyData, pCurWin->pStateKey->bytes);

×

4620

  *pWinCode = winCode;

×

4622

  qDebug("===stream===set state cur win buff. skey:%" PRId64 ", endkey:%" PRId64, pCurWin->winInfo.sessionWin.win.skey,

×

4625

  pNextWin->winInfo.sessionWin = pCurWin->winInfo.sessionWin;

×

4626

  getNextStateWin(pAggSup, pNextWin, true);

×

4627

  qDebug("===stream===set state next win buff. skey:%" PRId64 ", endkey:%" PRId64,

×

4629

_end:

×

4630

  if (code != TSDB_CODE_SUCCESS) {

×

4631

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

4633

  return code;

×

4636

int32_t updateStateWindowInfo(SStreamAggSupporter* pAggSup, SStateWindowInfo* pWinInfo, SStateWindowInfo* pNextWin,

×

4640

  int32_t code = TSDB_CODE_SUCCESS;

×

4641

  int32_t lino = 0;

×

4642

  *allEqual = true;

×

4643

  for (int32_t i = start; i < rows; ++i) {

×

4644

    char* pKeyData = colDataGetData(pKeyCol, i);

×

4645

    if (!isTsInWindow(pWinInfo, pTs[i])) {

×

4646

      if (isEqualStateKey(pWinInfo, pKeyData)) {

×

4647

        if (IS_VALID_SESSION_WIN(pNextWin->winInfo)) {

×

4649

          if (pTs[i] >= pNextWin->winInfo.sessionWin.win.skey) {

×

4650

            (*pWinRows) = i - start;

×

4651

            goto _end;

×

4655

        (*pWinRows) = i - start;

×

4656

        goto _end;

×

4660

    if (pWinInfo->winInfo.sessionWin.win.skey > pTs[i]) {

×

4661

      if (pSeDeleted && pWinInfo->winInfo.isOutput) {

×

4662

        code = saveDeleteRes(pSeDeleted, pWinInfo->winInfo.sessionWin);

×

4663

        QUERY_CHECK_CODE(code, lino, _end);

×

4665

      removeSessionResult(pAggSup, pSeUpdated, pResultRows, &pWinInfo->winInfo.sessionWin);

×

4666

      pWinInfo->winInfo.sessionWin.win.skey = pTs[i];

×

4668

    pWinInfo->winInfo.sessionWin.win.ekey = TMAX(pWinInfo->winInfo.sessionWin.win.ekey, pTs[i]);

×

4669

    memcpy(pWinInfo->winInfo.pStatePos->pKey, &pWinInfo->winInfo.sessionWin, sizeof(SSessionKey));

×

4670

    if (!isEqualStateKey(pWinInfo, pKeyData)) {

×

4671

      *allEqual = false;

×

4674

  (*pWinRows) = rows - start;

×

4676

_end:

×

4677

  if (code != TSDB_CODE_SUCCESS) {

×

4678

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

4680

  return code;

×

4683

static bool isWinResult(SSessionKey* pKey, SSHashObj* pSeUpdate, SSHashObj* pResults) {

×

4684

  SSessionKey checkKey = {0};

×

4685

  getSessionHashKey(pKey, &checkKey);

×

4686

  if (tSimpleHashGet(pSeUpdate, &checkKey, sizeof(SSessionKey)) != NULL) {

×

4687

    return true;

×

4690

  if (tSimpleHashGet(pResults, &checkKey, sizeof(SSessionKey)) != NULL) {

×

4691

    return true;

×

4693

  return false;

×

4696

static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pSeUpdated,

×

4698

  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;

×

4699

  SStorageAPI*   pAPI = &pOperator->pTaskInfo->storageAPI;

×

4701

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

4702

  int32_t                      numOfOutput = pOperator->exprSupp.numOfExprs;

×

4703

  uint64_t                     groupId = pSDataBlock->info.id.groupId;

×

4704

  int32_t                      code = TSDB_CODE_SUCCESS;

×

4705

  int32_t                      lino = 0;

×

4706

  TSKEY*                       tsCols = NULL;

×

4707

  SResultRow*                  pResult = NULL;

×

4708

  int32_t                      winRows = 0;

×

4709

  SStreamAggSupporter*         pAggSup = &pInfo->streamAggSup;

×

4710

  SStreamNotifyEventSupp*      pNotifySup = &pInfo->basic.notifyEventSup;

×

4711

  STaskNotifyEventStat*        pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;

×

4713

  pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);

×

4714

  pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow;

×

4715

  if (pAggSup->winRange.ekey <= 0) {

×

4716

    pAggSup->winRange.ekey = INT64_MAX;

×

4719

  if (pSDataBlock->pDataBlock != NULL) {

×

4720

    SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);

×

4721

    if (!pColDataInfo) {

×

4722

      code = TSDB_CODE_FAILED;

×

4723

      QUERY_CHECK_CODE(code, lino, _end);

×

4725

    tsCols = (int64_t*)pColDataInfo->pData;

×

4727

    return;

×

4730

  int32_t rows = pSDataBlock->info.rows;

×

4731

  code = blockDataEnsureCapacity(pAggSup->pScanBlock, rows);

×

4732

  QUERY_CHECK_CODE(code, lino, _end);

×

4734

  SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);

×

4735

  for (int32_t i = 0; i < rows; i += winRows) {

×

4736

    if (pInfo->ignoreExpiredData && checkExpiredData(&pInfo->streamAggSup.stateStore, pInfo->streamAggSup.pUpdateInfo,

×

4737

                                                     &pInfo->twAggSup, pSDataBlock->info.id.uid, tsCols[i], NULL, 0) ||

×

4738

        colDataIsNull_s(pKeyColInfo, i)) {

×

4739

      i++;

×

4740

      continue;

×

4742

    char*            pKeyData = colDataGetData(pKeyColInfo, i);

×

4743

    int32_t          winIndex = 0;

×

4744

    bool             allEqual = true;

×

4745

    SStateWindowInfo curWin = {0};

×

4746

    SStateWindowInfo nextWin = {0};

×

4747

    int32_t          winCode = TSDB_CODE_SUCCESS;

×

4748

    code = setStateOutputBuf(pAggSup, tsCols[i], groupId, pKeyData, &curWin, &nextWin, &winCode);

×

4749

    QUERY_CHECK_CODE(code, lino, _end);

×

4751

    if (winCode != TSDB_CODE_SUCCESS && pTaskInfo->streamInfo.eventTypes) {

×

4752

      SStateWindowInfo prevWin = {.winInfo.sessionWin = curWin.winInfo.sessionWin};

×

4753

      getNextStateWin(pAggSup, &prevWin, false);

×

4754

      qDebug("===stream===get state prev win buff. skey:%" PRId64 ", endkey:%" PRId64,

×

4756

      releaseOutputBuf(pAggSup->pState, prevWin.winInfo.pStatePos, &pAPI->stateStore);

×

4759

      if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE) &&

×

4760

          IS_VALID_SESSION_WIN(prevWin.winInfo)) {

×

4761

        code = addStateAggNotifyEvent(SNOTIFY_EVENT_WINDOW_CLOSE, &prevWin.winInfo.sessionWin, prevWin.pStateKey,

×

4762

                                      curWin.pStateKey, true, pNotifySup, pNotifyEventStat);

×

4763

        QUERY_CHECK_CODE(code, lino, _end);

×

4765

      if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_OPEN)) {

×

4766

        code = addStateAggNotifyEvent(SNOTIFY_EVENT_WINDOW_OPEN, &curWin.winInfo.sessionWin, curWin.pStateKey,

×

4767

                                      prevWin.pStateKey, false, pNotifySup, pNotifyEventStat);

×

4768

        QUERY_CHECK_CODE(code, lino, _end);

×

4772

    if (isWinResult(&nextWin.winInfo.sessionWin, pSeUpdated, pAggSup->pResultRows) == false) {

×

4773

      releaseOutputBuf(pAggSup->pState, nextWin.winInfo.pStatePos, &pAPI->stateStore);

×

4776

    setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo);

×

4777

    code = updateStateWindowInfo(pAggSup, &curWin, &nextWin, tsCols, groupId, pKeyColInfo, rows, i, &allEqual,

×

4779

    QUERY_CHECK_CODE(code, lino, _end);

×

4781

    if (!allEqual) {

×

4782

      uint64_t uid = 0;

×

4783

      code = appendDataToSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey,

×

4785

      QUERY_CHECK_CODE(code, lino, _end);

×

4786

      int32_t tmpRes = tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey));

×

4787

      qTrace("%s at line %d res:%d", __func__, __LINE__, tmpRes);

×

4789

      doDeleteSessionWindow(pAggSup, &curWin.winInfo.sessionWin);

×

4790

      releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore);

×

4791

      continue;

×

4794

    code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput,

×

4796

    QUERY_CHECK_CODE(code, lino, _end);

×

4798

    code = saveSessionOutputBuf(pAggSup, &curWin.winInfo);

×

4799

    QUERY_CHECK_CODE(code, lino, _end);

×

4801

    if (pInfo->destHasPrimaryKey && curWin.winInfo.isOutput && IS_NORMAL_STATE_OP(pOperator)) {

×

4802

      code = saveDeleteRes(pInfo->pPkDeleted, curWin.winInfo.sessionWin);

×

4803

      QUERY_CHECK_CODE(code, lino, _end);

×

4806

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {

×

4807

      code = saveResult(curWin.winInfo, pSeUpdated);

×

4808

      QUERY_CHECK_CODE(code, lino, _end);

×

4811

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

4812

      curWin.winInfo.pStatePos->beUpdated = true;

×

4813

      SSessionKey key = {0};

×

4814

      getSessionHashKey(&curWin.winInfo.sessionWin, &key);

×

4816

          tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo));

×

4817

      QUERY_CHECK_CODE(code, lino, _end);

×

4822

    if (BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE)) {

×

4823

      code = addStateAggNotifyEvent(SNOTIFY_EVENT_WINDOW_CLOSE, &curWin.winInfo.sessionWin, curWin.pStateKey,

×

4824

                                    nextWin.pStateKey, false, pNotifySup, pNotifyEventStat);

×

4825

      QUERY_CHECK_CODE(code, lino, _end);

×

4829

_end:

×

4830

  if (code != TSDB_CODE_SUCCESS) {

×

4831

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

4835

int32_t doStreamStateEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOperator, bool isParent) {

×

4836

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

4837

  if (!pInfo) {

×

4838

    return 0;

×

4841

  void* pData = (buf == NULL) ? NULL : *buf;

×

4844

  int32_t tlen = 0;

×

4845

  int32_t mapSize = tSimpleHashGetSize(pInfo->streamAggSup.pResultRows);

×

4846

  tlen += taosEncodeFixedI32(buf, mapSize);

×

4847

  void*   pIte = NULL;

×

4848

  size_t  keyLen = 0;

×

4849

  int32_t iter = 0;

×

4850

  while ((pIte = tSimpleHashIterate(pInfo->streamAggSup.pResultRows, pIte, &iter)) != NULL) {

×

4851

    void* key = tSimpleHashGetKey(pIte, &keyLen);

×

4852

    tlen += encodeSSessionKey(buf, key);

×

4853

    tlen += encodeSResultWindowInfo(buf, pIte, pInfo->streamAggSup.resultRowSize);

×

4857

  tlen += encodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);

×

4860

  int32_t size = taosArrayGetSize(pInfo->pChildren);

×

4861

  tlen += taosEncodeFixedI32(buf, size);

×

4862

  for (int32_t i = 0; i < size; i++) {

×

4863

    SOperatorInfo* pChOp = taosArrayGetP(pInfo->pChildren, i);

×

4864

    tlen += doStreamStateEncodeOpState(buf, 0, pChOp, false);

×

4868

  tlen += taosEncodeFixedI64(buf, pInfo->dataVersion);

×

4871

  tlen += encodeStreamBasicInfo(buf, &pInfo->basic);

×

4874

  if (isParent) {

×

4875

    if (buf) {

×

4876

      uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t));

×

4877

      tlen += taosEncodeFixedU32(buf, cksum);

×

4879

      tlen += sizeof(uint32_t);

×

4883

  return tlen;

×

4886

int32_t doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator, bool isParent, void** ppBuf) {

×

4887

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

4888

  int32_t                      code = TSDB_CODE_SUCCESS;

×

4889

  int32_t                      lino = 0;

×

4890

  SStreamAggSupporter*         pAggSup = &pInfo->streamAggSup;

×

4891

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

4892

  void*                        pDataEnd = POINTER_SHIFT(buf, len);

×

4893

  if (!pInfo) {

×

4894

    code = TSDB_CODE_FAILED;

×

4895

    QUERY_CHECK_CODE(code, lino, _end);

×

4899

  if (isParent) {

×

4900

    int32_t dataLen = len - sizeof(uint32_t);

×

4901

    void*   pCksum = POINTER_SHIFT(buf, dataLen);

×

4902

    if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) {

×

4903

      qError("stream state_window state is invalid");

×

4904

      code = TSDB_CODE_FAILED;

×

4905

      QUERY_CHECK_CODE(code, lino, _end);

×

4907

    pDataEnd = pCksum;

×

4911

  int32_t mapSize = 0;

×

4912

  buf = taosDecodeFixedI32(buf, &mapSize);

×

4913

  for (int32_t i = 0; i < mapSize; i++) {

×

4914

    SResultWindowInfo winfo = {0};

×

4915

    buf = decodeSSessionKey(buf, &winfo.sessionWin);

×

4916

    int32_t winCode = TSDB_CODE_SUCCESS;

×

4917

    code = pAggSup->stateStore.streamStateStateAddIfNotExist(

×

4920

    QUERY_CHECK_CODE(code, lino, _end);

×

4922

    buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize);

×

4923

    code = tSimpleHashPut(pInfo->streamAggSup.pResultRows, &winfo.sessionWin, sizeof(SSessionKey), &winfo,

×

4925

    QUERY_CHECK_CODE(code, lino, _end);

×

4929

  buf = decodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);

×

4932

  int32_t size = 0;

×

4933

  buf = taosDecodeFixedI32(buf, &size);

×

4934

  for (int32_t i = 0; i < size; i++) {

×

4935

    SOperatorInfo* pChOp = taosArrayGetP(pInfo->pChildren, i);

×

4936

    code = doStreamStateDecodeOpState(buf, 0, pChOp, false, &buf);

×

4937

    QUERY_CHECK_CODE(code, lino, _end);

×

4941

  buf = taosDecodeFixedI64(buf, &pInfo->dataVersion);

×

4943

  if (ppBuf) {

×

4944

    (*ppBuf) = buf;

×

4948

  if (buf < pDataEnd) {

×

4949

    code = decodeStreamBasicInfo(&buf, &pInfo->basic);

×

4950

    QUERY_CHECK_CODE(code, lino, _end);

×

4953

_end:

×

4954

  if (code != TSDB_CODE_SUCCESS) {

×

4955

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

4957

  return code;

×

4960

void doStreamStateSaveCheckpoint(SOperatorInfo* pOperator) {

×

4961

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

4962

  if (needSaveStreamOperatorInfo(&pInfo->basic)) {

×

4963

    int32_t len = doStreamStateEncodeOpState(NULL, 0, pOperator, true);

×

4964

    void*   buf = taosMemoryCalloc(1, len);

×

4965

    if (!buf) {

×

4966

      qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_OUT_OF_MEMORY));

×

4967

      return;

×

4969

    void* pBuf = buf;

×

4970

    len = doStreamStateEncodeOpState(&pBuf, len, pOperator, true);

×

4971

    pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_CHECKPOINT_NAME,

×

4973

    taosMemoryFree(buf);

×

4974

    saveStreamOperatorStateComplete(&pInfo->basic);

×

4978

static int32_t buildStateResult(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

4979

  int32_t                      code = TSDB_CODE_SUCCESS;

×

4980

  int32_t                      lino = 0;

×

4981

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

4982

  SOptrBasicInfo*              pBInfo = &pInfo->binfo;

×

4983

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

4984

  SStreamNotifyEventSupp*      pNotifySup = &pInfo->basic.notifyEventSup;

×

4985

  STaskNotifyEventStat*        pNotifyEventStat = pTaskInfo->streamInfo.pNotifyEventStat;

×

4986

  bool                         addNotifyEvent = false;

×

4987

  addNotifyEvent = BIT_FLAG_TEST_MASK(pTaskInfo->streamInfo.eventTypes, SNOTIFY_EVENT_WINDOW_CLOSE);

×

4988

  doBuildDeleteDataBlock(pOperator, pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator, &pInfo->groupResInfo);

×

4989

  if (pInfo->pDelRes->info.rows > 0) {

×

4990

    printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

4991

    if (addNotifyEvent) {

×

4992

      code = addAggDeleteNotifyEvent(pInfo->pDelRes, pNotifySup, pNotifyEventStat);

×

4993

      QUERY_CHECK_CODE(code, lino, _end);

×

4995

    (*ppRes) = pInfo->pDelRes;

×

4996

    return code;

×

4999

  doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes,

×

5001

  if (pBInfo->pRes->info.rows > 0) {

×

5002

    printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5003

    if (addNotifyEvent) {

×

5004

      code = addAggResultNotifyEvent(pBInfo->pRes, pNotifySup->pSessionKeys, pTaskInfo->streamInfo.notifyResultSchema,

×

5006

      QUERY_CHECK_CODE(code, lino, _end);

×

5008

    (*ppRes) = pBInfo->pRes;

×

5009

    return code;

×

5012

  code = buildNotifyEventBlock(pTaskInfo, pNotifySup, pNotifyEventStat);

×

5013

  QUERY_CHECK_CODE(code, lino, _end);

×

5014

  if (pNotifySup->pEventBlock && pNotifySup->pEventBlock->info.rows > 0) {

×

5015

    printDataBlock(pNotifySup->pEventBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5016

    (*ppRes) = pNotifySup->pEventBlock;

×

5017

    return code;

×

5020

  code = removeOutdatedNotifyEvents(&pInfo->twAggSup, pNotifySup, pNotifyEventStat);

×

5021

  QUERY_CHECK_CODE(code, lino, _end);

×

5023

_end:

×

5024

  if (code != TSDB_CODE_SUCCESS) {

×

5025

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

5027

  (*ppRes) = NULL;

×

5028

  return code;

×

5031

static int32_t doStreamStateAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

5032

  if (pOperator->status == OP_EXEC_DONE) {

×

5033

    (*ppRes) = NULL;

×

5034

    return TSDB_CODE_SUCCESS;

×

5037

  int32_t                      code = TSDB_CODE_SUCCESS;

×

5038

  int32_t                      lino = 0;

×

5039

  SExprSupp*                   pSup = &pOperator->exprSupp;

×

5040

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

5041

  SOptrBasicInfo*              pBInfo = &pInfo->binfo;

×

5042

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

5043

  qDebug("===stream=== stream state agg");

×

5044

  if (pOperator->status == OP_RES_TO_RETURN) {

×

5045

    SSDataBlock* resBlock = NULL;

×

5046

    code = buildStateResult(pOperator, &resBlock);

×

5047

    QUERY_CHECK_CODE(code, lino, _end);

×

5048

    if (resBlock != NULL) {

×

5049

      (*ppRes) = resBlock;

×

5050

      return code;

×

5053

    if (pInfo->recvGetAll) {

×

5054

      pInfo->recvGetAll = false;

×

5055

      resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows);

×

5058

    if (pInfo->reCkBlock) {

×

5059

      pInfo->reCkBlock = false;

×

5060

      printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5061

      (*ppRes) = pInfo->pCheckpointRes;

×

5062

      return code;

×

5065

    setStreamOperatorCompleted(pOperator);

×

5066

    (*ppRes) = NULL;

×

5067

    return code;

×

5070

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

5071

  if (!pInfo->pUpdated) {

×

5072

    pInfo->pUpdated = taosArrayInit(16, sizeof(SResultWindowInfo));

×

5073

    QUERY_CHECK_NULL(pInfo->pUpdated, code, lino, _end, terrno);

×

5075

  if (!pInfo->pSeUpdated) {

×

5076

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

5077

    pInfo->pSeUpdated = tSimpleHashInit(64, hashFn);

×

5078

    QUERY_CHECK_NULL(pInfo->pSeUpdated, code, lino, _end, terrno);

×

5080

  while (1) {

×

5081

    SSDataBlock* pBlock = NULL;

×

5082

    code = downstream->fpSet.getNextFn(downstream, &pBlock);

×

5083

    QUERY_CHECK_CODE(code, lino, _end);

×

5085

    if (pBlock == NULL) {

×

5086

      break;

×

5088

    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));

×

5089

    setStreamOperatorState(&pInfo->basic, pBlock->info.type);

×

5091

    if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||

×

5092

        pBlock->info.type == STREAM_CLEAR) {

×

5093

      bool add = pInfo->destHasPrimaryKey && IS_NORMAL_STATE_OP(pOperator);

×

5094

      code = deleteSessionWinState(&pInfo->streamAggSup, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted,

×

5096

      QUERY_CHECK_CODE(code, lino, _end);

×

5097

      continue;

×

5098

    } else if (pBlock->info.type == STREAM_GET_ALL) {

×

5099

      pInfo->recvGetAll = true;

×

5100

      code = getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pSeUpdated);

×

5101

      QUERY_CHECK_CODE(code, lino, _end);

×

5102

      continue;

×

5103

    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {

×

5104

      (*ppRes) = pBlock;

×

5105

      return code;

×

5106

    } else if (pBlock->info.type == STREAM_CHECKPOINT) {

×

5107

      pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState);

×

5108

      doStreamStateSaveCheckpoint(pOperator);

×

5109

      code = copyDataBlock(pInfo->pCheckpointRes, pBlock);

×

5110

      QUERY_CHECK_CODE(code, lino, _end);

×

5112

      continue;

×

5114

      if (pBlock->info.type != STREAM_NORMAL && pBlock->info.type != STREAM_INVALID) {

×

5115

        code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;

×

5116

        QUERY_CHECK_CODE(code, lino, _end);

×

5120

    if (pInfo->scalarSupp.pExprInfo != NULL) {

×

5121

      SExprSupp* pExprSup = &pInfo->scalarSupp;

×

5122

      code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);

×

5123

      QUERY_CHECK_CODE(code, lino, _end);

×

5126

    code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

5127

    QUERY_CHECK_CODE(code, lino, _end);

×

5128

    doStreamStateAggImpl(pOperator, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted);

×

5129

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);

×

5132

  pOperator->status = OP_RES_TO_RETURN;

×

5134

  code = closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pInfo->pSeUpdated);

×

5135

  QUERY_CHECK_CODE(code, lino, _end);

×

5137

  code = copyUpdateResult(&pInfo->pSeUpdated, pInfo->pUpdated, sessionKeyCompareAsc);

×

5138

  QUERY_CHECK_CODE(code, lino, _end);

×

5140

  removeSessionDeleteResults(pInfo->pSeDeleted, pInfo->pUpdated);

×

5142

  if (pInfo->isHistoryOp) {

×

5143

    code = getMaxTsWins(pInfo->pUpdated, pInfo->historyWins);

×

5144

    QUERY_CHECK_CODE(code, lino, _end);

×

5146

  if (pInfo->destHasPrimaryKey && IS_NORMAL_STATE_OP(pOperator)) {

×

5147

    code = copyDeleteSessionKey(pInfo->pPkDeleted, pInfo->pSeDeleted);

×

5148

    QUERY_CHECK_CODE(code, lino, _end);

×

5151

  initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);

×

5152

  pInfo->pUpdated = NULL;

×

5153

  code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);

×

5154

  QUERY_CHECK_CODE(code, lino, _end);

×

5156

  SSDataBlock* resBlock = NULL;

×

5157

  code = buildStateResult(pOperator, &resBlock);

×

5158

  QUERY_CHECK_CODE(code, lino, _end);

×

5159

  if (resBlock != NULL) {

×

5160

    (*ppRes) = resBlock;

×

5161

    return code;

×

5164

_end:

×

5165

  if (code != TSDB_CODE_SUCCESS) {

×

5166

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

5167

    pTaskInfo->code = code;

×

5168

    T_LONG_JMP(pTaskInfo->env, code);

×

5170

  setStreamOperatorCompleted(pOperator);

×

5171

  (*ppRes) = NULL;

×

5172

  return code;

×

5175

static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {

×

5176

  SSDataBlock* pRes = NULL;

×

5177

  int32_t      code = doStreamStateAggNext(pOperator, &pRes);

×

5178

  return pRes;

×

5181

void streamStateReleaseState(SOperatorInfo* pOperator) {

×

5182

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

5183

  int32_t                      winSize = taosArrayGetSize(pInfo->historyWins) * sizeof(SSessionKey);

×

5184

  int32_t                      resSize = winSize + sizeof(TSKEY);

×

5185

  char*                        pBuff = taosMemoryCalloc(1, resSize);

×

5186

  if (!pBuff) {

×

5187

    return;

×

5189

  memcpy(pBuff, pInfo->historyWins->pData, winSize);

×

5190

  memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY));

×

5191

  qDebug("===stream=== relase state. save result count:%d", (int32_t)taosArrayGetSize(pInfo->historyWins));

×

5192

  pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_STATE_NAME,

×

5194

  pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState);

×

5195

  taosMemoryFreeClear(pBuff);

×

5197

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

5198

  if (downstream->fpSet.releaseStreamStateFn) {

×

5199

    downstream->fpSet.releaseStreamStateFn(downstream);

×

5203

static int32_t compactStateWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin,

×

5205

  SExprSupp*                   pSup = &pOperator->exprSupp;

×

5206

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

5207

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

5208

  return compactTimeWindow(pSup, &pInfo->streamAggSup, &pInfo->twAggSup, pTaskInfo, pCurWin, pNextWin, pStUpdated,

×

5212

void streamStateReloadState(SOperatorInfo* pOperator) {

×

5213

  int32_t                      code = TSDB_CODE_SUCCESS;

×

5214

  int32_t                      lino = 0;

×

5215

  SStreamStateAggOperatorInfo* pInfo = pOperator->info;

×

5216

  SStreamAggSupporter*         pAggSup = &pInfo->streamAggSup;

×

5217

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

5218

  resetWinRange(&pAggSup->winRange);

×

5220

  SSessionKey seKey = {.win.skey = INT64_MIN, .win.ekey = INT64_MIN, .groupId = 0};

×

5221

  int32_t     size = 0;

×

5222

  void*       pBuf = NULL;

×

5223

  code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_STATE_OP_STATE_NAME,

×

5225

  QUERY_CHECK_CODE(code, lino, _end);

×

5227

  int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey);

×

5228

  qDebug("===stream=== reload state. get result count:%d", num);

×

5229

  SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf;

×

5231

  TSKEY ts = *(TSKEY*)((char*)pBuf + size - sizeof(TSKEY));

×

5232

  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts);

×

5233

  pAggSup->stateStore.streamStateReloadInfo(pAggSup->pState, ts);

×

5235

  if (!pInfo->pSeUpdated && num > 0) {

×

5236

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

5237

    pInfo->pSeUpdated = tSimpleHashInit(64, hashFn);

×

5238

    QUERY_CHECK_NULL(pInfo->pSeUpdated, code, lino, _end, terrno);

×

5240

  if (!pInfo->pSeDeleted && num > 0) {

×

5241

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

5242

    pInfo->pSeDeleted = tSimpleHashInit(64, hashFn);

×

5243

    QUERY_CHECK_NULL(pInfo->pSeDeleted, code, lino, _end, terrno);

×

5245

  for (int32_t i = 0; i < num; i++) {

×

5246

    SStateWindowInfo curInfo = {0};

×

5247

    SStateWindowInfo nextInfo = {0};

×

5248

    qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey,

×

5250

    code = getStateWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo, &nextInfo);

×

5251

    QUERY_CHECK_CODE(code, lino, _end);

×

5253

    bool cpRes = compareWinStateKey(curInfo.pStateKey, nextInfo.pStateKey);

×

5254

    qDebug("===stream=== reload state. next window info %" PRId64 ", %" PRIu64 ", compare:%d",

×

5256

    if (cpRes) {

×

5257

      code = compactStateWindow(pOperator, &curInfo.winInfo, &nextInfo.winInfo, pInfo->pSeUpdated, pInfo->pSeDeleted);

×

5258

      qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey,

×

5260

      QUERY_CHECK_CODE(code, lino, _end);

×

5262

      if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {

×

5263

        code = saveResult(curInfo.winInfo, pInfo->pSeUpdated);

×

5264

        QUERY_CHECK_CODE(code, lino, _end);

×

5265

      } else if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

5266

        if (!isCloseWindow(&curInfo.winInfo.sessionWin.win, &pInfo->twAggSup)) {

×

5267

          code = saveDeleteRes(pInfo->pSeDeleted, curInfo.winInfo.sessionWin);

×

5268

          QUERY_CHECK_CODE(code, lino, _end);

×

5270

        SSessionKey key = {0};

×

5271

        getSessionHashKey(&curInfo.winInfo.sessionWin, &key);

×

5272

        code = tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curInfo.winInfo,

×

5274

        QUERY_CHECK_CODE(code, lino, _end);

×

5276

    } else if (IS_VALID_SESSION_WIN(nextInfo.winInfo)) {

×

5277

      releaseOutputBuf(pAggSup->pState, nextInfo.winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore);

×

5280

    if (IS_VALID_SESSION_WIN(curInfo.winInfo)) {

×

5281

      code = saveSessionOutputBuf(pAggSup, &curInfo.winInfo);

×

5282

      QUERY_CHECK_CODE(code, lino, _end);

×

5285

  taosMemoryFreeClear(pBuf);

×

5287

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

5288

  if (downstream->fpSet.reloadStreamStateFn) {

×

5289

    downstream->fpSet.reloadStreamStateFn(downstream);

×

5291

  reloadAggSupFromDownStream(downstream, &pInfo->streamAggSup);

×

5293

_end:

×

5294

  taosMemoryFreeClear(pBuf);

×

5295

  if (code != TSDB_CODE_SUCCESS) {

×

5296

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

5298

×

5300

int32_t createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,

×

5302

  QRY_PARAM_CHECK(pOptrInfo);

×

5303

  int32_t code = 0;

×

5304

  int32_t lino = 0;

×

5306

  SStreamStateWinodwPhysiNode* pStateNode = (SStreamStateWinodwPhysiNode*)pPhyNode;

×

5307

  int32_t                      tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;

×

5308

  SColumnNode*                 pColNode = (SColumnNode*)(pStateNode->pStateKey);

×

5309

  SStreamStateAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamStateAggOperatorInfo));

×

5310

  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));

×

5311

  if (pInfo == NULL || pOperator == NULL) {

×

5312

    code = terrno;

×

5313

    QUERY_CHECK_CODE(code, lino, _error);

×

5316

  pInfo->stateCol = extractColumnFromColumnNode(pColNode);

×

5317

  initResultSizeInfo(&pOperator->resultInfo, 4096);

×

5318

  if (pStateNode->window.pExprs != NULL) {

×

5319

    int32_t    numOfScalar = 0;

×

5320

    SExprInfo* pScalarExprInfo = NULL;

×

5321

    code = createExprInfo(pStateNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalar);

×

5322

    QUERY_CHECK_CODE(code, lino, _error);

×

5324

    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);

×

5325

    QUERY_CHECK_CODE(code, lino, _error);

×

5328

  pInfo->twAggSup = (STimeWindowAggSupp){

×

5329

      .waterMark = pStateNode->window.watermark,

×

5330

      .calTrigger = pStateNode->window.triggerType,

×

5333

      .deleteMark = getDeleteMark(&pStateNode->window, 0),

×

5336

  code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

×

5337

  QUERY_CHECK_CODE(code, lino, _error);

×

5339

  SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);

×

5340

  QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);

×

5341

  pInfo->binfo.pRes = pResBlock;

×

5343

  SExprSupp* pExpSup = &pOperator->exprSupp;

×

5344

  int32_t    numOfCols = 0;

×

5345

  SExprInfo* pExprInfo = NULL;

×

5346

  code = createExprInfo(pStateNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);

×

5347

  QUERY_CHECK_CODE(code, lino, _error);

×

5349

  code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore);

×

5350

  if (code != TSDB_CODE_SUCCESS) {

×

5351

    goto _error;

×

5353

  int32_t keySize = sizeof(SStateKeys) + pColNode->node.resType.bytes;

×

5354

  int16_t type = pColNode->node.resType.type;

×

5355

  pInfo->primaryTsIndex = tsSlotId;

×

5357

      initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState, keySize, type,

×

5358

                             &pTaskInfo->storageAPI.stateStore, pHandle, &pInfo->twAggSup, GET_TASKID(pTaskInfo),

×

5360

  QUERY_CHECK_CODE(code, lino, _error);

×

5362

  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

5363

  pInfo->pSeDeleted = tSimpleHashInit(64, hashFn);

×

5364

  QUERY_CHECK_NULL(pInfo->pSeDeleted, code, lino, _error, terrno);

×

5365

  pInfo->pDelIterator = NULL;

×

5367

  code = createSpecialDataBlock(STREAM_DELETE_RESULT, &pInfo->pDelRes);

×

5368

  QUERY_CHECK_CODE(code, lino, _error);

×

5370

  pInfo->pChildren = NULL;

×

5371

  pInfo->ignoreExpiredData = pStateNode->window.igExpired;

×

5372

  pInfo->ignoreExpiredDataSaved = false;

×

5373

  pInfo->pUpdated = NULL;

×

5374

  pInfo->pSeUpdated = NULL;

×

5375

  pInfo->dataVersion = 0;

×

5376

  pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey));

×

5377

  if (!pInfo->historyWins) {

×

5378

    code = terrno;

×

5379

    QUERY_CHECK_CODE(code, lino, _error);

×

5382

  if (pHandle) {

×

5383

    pInfo->isHistoryOp = pHandle->fillHistory;

×

5386

  code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);

×

5387

  QUERY_CHECK_CODE(code, lino, _error);

×

5389

  pInfo->recvGetAll = false;

×

5390

  pInfo->pPkDeleted = tSimpleHashInit(64, hashFn);

×

5391

  QUERY_CHECK_NULL(pInfo->pPkDeleted, code, lino, _error, terrno);

×

5392

  pInfo->destHasPrimaryKey = pStateNode->window.destHasPrimaryKey;

×

5393

  pInfo->pOperator = pOperator;

×

5395

  setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED,

×

5398

  void*   buff = NULL;

×

5399

  int32_t len = 0;

×

5401

      pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_STATE_OP_CHECKPOINT_NAME,

×

5403

  if (res == TSDB_CODE_SUCCESS) {

×

5404

    code = doStreamStateDecodeOpState(buff, len, pOperator, true, NULL);

×

5405

    taosMemoryFree(buff);

×

5406

    QUERY_CHECK_CODE(code, lino, _error);

×

5409

  pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAggNext, NULL, destroyStreamStateOperatorInfo,

×

5411

  setOperatorStreamStateFn(pOperator, streamStateReleaseState, streamStateReloadState);

×

5413

  code = initStreamBasicInfo(&pInfo->basic, pOperator);

×

5414

  QUERY_CHECK_CODE(code, lino, _error);

×

5416

  code = initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex,

×

5417

                        &pInfo->twAggSup, &pInfo->basic);

×

5418

  QUERY_CHECK_CODE(code, lino, _error);

×

5420

  code = appendDownstream(pOperator, &downstream, 1);

×

5421

  QUERY_CHECK_CODE(code, lino, _error);

×

5423

  pInfo->trueForLimit = pStateNode->trueForLimit;

×

5425

  *pOptrInfo = pOperator;

×

5426

  return TSDB_CODE_SUCCESS;

×

5428

_error:

×

5429

  if (pInfo != NULL) destroyStreamStateOperatorInfo(pInfo);

×

5430

  destroyOperatorAndDownstreams(pOperator, &downstream, 1);

×

5431

  pTaskInfo->code = code;

×

5432

  qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

5433

  return code;

×

5448

static int32_t doStreamIntervalAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

5449

  int32_t                      code = TSDB_CODE_SUCCESS;

×

5450

  int32_t                      lino = 0;

×

5451

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

5452

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

5453

  SStorageAPI*                 pAPI = &pOperator->pTaskInfo->storageAPI;

×

5454

  SExprSupp*                   pSup = &pOperator->exprSupp;

×

5456

  qDebug("stask:%s  %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);

×

5458

  if (pOperator->status == OP_EXEC_DONE) {

×

5459

    (*ppRes) = NULL;

×

5460

    return code;

×

5463

  if (pOperator->status == OP_RES_TO_RETURN) {

×

5464

    SSDataBlock* resBlock = NULL;

×

5465

    code = buildIntervalResult(pOperator, &resBlock);

×

5466

    QUERY_CHECK_CODE(code, lino, _end);

×

5467

    if (resBlock != NULL) {

×

5468

      (*ppRes) = resBlock;

×

5469

      return code;

×

5472

    if (pInfo->recvGetAll) {

×

5473

      pInfo->recvGetAll = false;

×

5474

      resetUnCloseWinInfo(pInfo->aggSup.pResultRowHashTable);

×

5477

    if (pInfo->reCkBlock) {

×

5478

      pInfo->reCkBlock = false;

×

5479

      printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5480

      (*ppRes) = pInfo->pCheckpointRes;

×

5481

      return code;

×

5484

    setStreamOperatorCompleted(pOperator);

×

5485

    (*ppRes) = NULL;

×

5486

    return code;

×

5489

  SOperatorInfo* downstream = pOperator->pDownstream[0];

×

5491

  if (!pInfo->pUpdated) {

×

5492

    pInfo->pUpdated = taosArrayInit(4096, POINTER_BYTES);

×

5493

    QUERY_CHECK_NULL(pInfo->pUpdated, code, lino, _end, terrno);

×

5496

  if (!pInfo->pUpdatedMap) {

×

5497

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

5498

    pInfo->pUpdatedMap = tSimpleHashInit(4096, hashFn);

×

5499

    QUERY_CHECK_NULL(pInfo->pUpdatedMap, code, lino, _end, terrno);

×

5502

  while (1) {

×

5503

    SSDataBlock* pBlock = NULL;

×

5504

    code = downstream->fpSet.getNextFn(downstream, &pBlock);

×

5505

    QUERY_CHECK_CODE(code, lino, _end);

×

5507

    if (pBlock == NULL) {

×

5508

      qDebug("===stream===return data:%s. recv datablock num:%" PRIu64, getStreamOpName(pOperator->operatorType),

×

5510

      pInfo->numOfDatapack = 0;

×

5511

      break;

×

5514

    pInfo->numOfDatapack++;

×

5515

    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));

×

5516

    setStreamOperatorState(&pInfo->basic, pBlock->info.type);

×

5518

    if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||

×

5519

        pBlock->info.type == STREAM_CLEAR) {

×

5520

      code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, pInfo->pDelWins, pInfo->pUpdatedMap, NULL);

×

5521

      QUERY_CHECK_CODE(code, lino, _end);

×

5522

      continue;

×

5523

    } else if (pBlock->info.type == STREAM_GET_ALL) {

×

5524

      pInfo->recvGetAll = true;

×

5525

      code = getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap);

×

5526

      QUERY_CHECK_CODE(code, lino, _end);

×

5527

      continue;

×

5528

    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {

×

5529

      printDataBlock(pBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5530

      (*ppRes) = pBlock;

×

5531

      return code;

×

5532

    } else if (pBlock->info.type == STREAM_DROP_CHILD_TABLE) {

×

5533

      doDeleteWindowByGroupId(pOperator, pBlock);

×

5534

      printDataBlock(pBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5535

      (*ppRes) = pBlock;

×

5536

      return code;

×

5537

    } else if (pBlock->info.type == STREAM_CHECKPOINT) {

×

5538

      pAPI->stateStore.streamStateCommit(pInfo->pState);

×

5539

      doStreamIntervalSaveCheckpoint(pOperator);

×

5540

      pInfo->reCkBlock = true;

×

5541

      code = copyDataBlock(pInfo->pCheckpointRes, pBlock);

×

5542

      QUERY_CHECK_CODE(code, lino, _end);

×

5544

      continue;

×

5546

      if (pBlock->info.type != STREAM_NORMAL && pBlock->info.type != STREAM_INVALID) {

×

5547

        code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;

×

5548

        QUERY_CHECK_CODE(code, lino, _end);

×

5552

    if (pBlock->info.type == STREAM_NORMAL && pBlock->info.version != 0) {

×

5554

      pTaskInfo->version = pBlock->info.version;

×

5557

    if (pInfo->scalarSupp.pExprInfo != NULL) {

×

5558

      SExprSupp* pExprSup = &pInfo->scalarSupp;

×

5559

      code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);

×

5560

      QUERY_CHECK_CODE(code, lino, _end);

×

5566

    code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

5567

    QUERY_CHECK_CODE(code, lino, _end);

×

5574

    code = doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pInfo->pUpdatedMap, pInfo->pDeletedMap);

×

5575

    if (code == TSDB_CODE_STREAM_INTERNAL_ERROR) {

×

5576

      pOperator->status = OP_RES_TO_RETURN;

×

5577

      code = TSDB_CODE_SUCCESS;

×

5578

      break;

×

5580

    QUERY_CHECK_CODE(code, lino, _end);

×

5581

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);

×

5582

    pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey);

×

5584

  pOperator->status = OP_RES_TO_RETURN;

×

5585

  if (!pInfo->destHasPrimaryKey) {

×

5586

    removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins);

×

5588

  code = closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL,

×

5590

  QUERY_CHECK_CODE(code, lino, _end);

×

5592

  if (pInfo->destHasPrimaryKey && IS_NORMAL_INTERVAL_OP(pOperator)) {

×

5593

    code = copyIntervalDeleteKey(pInfo->pDeletedMap, pInfo->pDelWins);

×

5594

    QUERY_CHECK_CODE(code, lino, _end);

×

5597

  void*   pIte = NULL;

×

5598

  int32_t iter = 0;

×

5599

  while ((pIte = tSimpleHashIterate(pInfo->pUpdatedMap, pIte, &iter)) != NULL) {

×

5600

    void* tmp = taosArrayPush(pInfo->pUpdated, pIte);

×

5601

    if (!tmp) {

×

5602

      code = terrno;

×

5603

      QUERY_CHECK_CODE(code, lino, _end);

×

5606

  taosArraySort(pInfo->pUpdated, winPosCmprImpl);

×

5608

  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);

×

5609

  pInfo->pUpdated = NULL;

×

5610

  code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);

×

5611

  QUERY_CHECK_CODE(code, lino, _end);

×

5613

  tSimpleHashCleanup(pInfo->pUpdatedMap);

×

5614

  pInfo->pUpdatedMap = NULL;

×

5616

  code = buildIntervalResult(pOperator, ppRes);

×

5617

  QUERY_CHECK_CODE(code, lino, _end);

×

5619

  return code;

×

5621

_end:

×

5622

  if (code != TSDB_CODE_SUCCESS) {

×

5623

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

5624

    pTaskInfo->code = code;

×

5625

    T_LONG_JMP(pTaskInfo->env, code);

×

5627

  setStreamOperatorCompleted(pOperator);

×

5628

  (*ppRes) = NULL;

×

5629

  return code;

×

5632

static int32_t createStreamSingleIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,

×

5635

  QRY_PARAM_CHECK(pOptrInfo);

×

5637

  int32_t code = TSDB_CODE_SUCCESS;

×

5638

  int32_t lino = 0;

×

5639

  int32_t numOfCols = 0;

×

5641

  SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo));

×

5642

  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));

×

5643

  if (pInfo == NULL || pOperator == NULL) {

×

5644

    code = terrno;

×

5645

    QUERY_CHECK_CODE(code, lino, _error);

×

5648

  SStreamIntervalPhysiNode* pIntervalPhyNode = (SStreamIntervalPhysiNode*)pPhyNode;

×

5650

  SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);

×

5651

  QUERY_CHECK_NULL(pResBlock, code, lino, _error, terrno);

×

5652

  initBasicInfo(&pInfo->binfo, pResBlock);

×

5654

  pInfo->interval = (SInterval){

×

5655

      .interval = pIntervalPhyNode->interval,

×

5656

      .sliding = pIntervalPhyNode->sliding,

×

5657

      .intervalUnit = pIntervalPhyNode->intervalUnit,

×

5658

      .slidingUnit = pIntervalPhyNode->slidingUnit,

×

5659

      .offset = pIntervalPhyNode->offset,

×

5660

      .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision,

×

5661

      .timeRange = pIntervalPhyNode->timeRange,

×

5663

  calcIntervalAutoOffset(&pInfo->interval);

×

5665

  pInfo->twAggSup =

×

5666

      (STimeWindowAggSupp){.waterMark = pIntervalPhyNode->window.watermark,

×

5667

                           .calTrigger = pIntervalPhyNode->window.triggerType,

×

5670

                           .deleteMark = getDeleteMark(&pIntervalPhyNode->window, pIntervalPhyNode->interval)};

×

5672

  pOperator->pTaskInfo = pTaskInfo;

×

5673

  SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI;

×

5675

  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;

×

5676

  pInfo->ignoreExpiredDataSaved = false;

×

5678

  SExprSupp* pSup = &pOperator->exprSupp;

×

5679

  pSup->hasWindowOrGroup = true;

×

5681

  code = initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);

×

5682

  QUERY_CHECK_CODE(code, lino, _error);

×

5684

  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;

×

5685

  initResultSizeInfo(&pOperator->resultInfo, 4096);

×

5687

  pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState));

×

5688

  QUERY_CHECK_NULL(pInfo->pState, code, lino, _error, terrno);

×

5689

  *(pInfo->pState) = *(pTaskInfo->streamInfo.pState);

×

5690

  pAPI->stateStore.streamStateSetNumber(pInfo->pState, -1, pInfo->primaryTsIndex);

×

5692

  size_t     keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;

×

5693

  SExprInfo* pExprInfo = NULL;

×

5694

  code = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);

×

5695

  QUERY_CHECK_CODE(code, lino, _error);

×

5696

  code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str, pInfo->pState,

×

5698

  QUERY_CHECK_CODE(code, lino, _error);

×

5700

  if (pIntervalPhyNode->window.pExprs != NULL) {

×

5701

    int32_t    numOfScalar = 0;

×

5702

    SExprInfo* pScalarExprInfo = NULL;

×

5704

    code = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalar);

×

5705

    QUERY_CHECK_CODE(code, lino, _error);

×

5707

    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);

×

5708

    QUERY_CHECK_CODE(code, lino, _error);

×

5711

  pInfo->invertible = false;

×

5712

  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));

×

5713

  QUERY_CHECK_NULL(pInfo->pDelWins, code, lino, _error, terrno);

×

5714

  pInfo->delIndex = 0;

×

5716

  code = createSpecialDataBlock(STREAM_DELETE_RESULT, &pInfo->pDelRes);

×

5717

  QUERY_CHECK_CODE(code, lino, _error);

×

5719

  initResultRowInfo(&pInfo->binfo.resultRowInfo);

×

5721

  pInfo->pPhyNode = NULL;  // create new child

×

5722

  pInfo->pPullDataMap = NULL;

×

5723

  pInfo->pFinalPullDataMap = NULL;

×

5724

  pInfo->pPullWins = NULL;  // SPullWindowInfo

×

5725

  pInfo->pullIndex = 0;

×

5726

  pInfo->pPullDataRes = NULL;

×

5727

  pInfo->numOfChild = 0;

×

5728

  pInfo->delKey.ts = INT64_MAX;

×

5729

  pInfo->delKey.groupId = 0;

×

5730

  pInfo->numOfDatapack = 0;

×

5731

  pInfo->pUpdated = NULL;

×

5732

  pInfo->pUpdatedMap = NULL;

×

5733

  int32_t funResSize = getMaxFunResSize(pSup, numOfCols);

×

5735

  pInfo->stateStore = pTaskInfo->storageAPI.stateStore;

×

5736

  pInfo->pState->pFileState = NULL;

×

5739

  pInfo->pState->pResultRowStore.resultRowGet = getResultRowFromBuf;

×

5740

  pInfo->pState->pResultRowStore.resultRowPut = putResultRowToBuf;

×

5741

  pInfo->pState->pExprSupp = &pOperator->exprSupp;

×

5743

  code = pTaskInfo->storageAPI.stateStore.streamFileStateInit(

×

5744

      tsStreamBufferSize, sizeof(SWinKey), pInfo->aggSup.resultRowSize, funResSize, compareTs, pInfo->pState,

×

5745

      pInfo->twAggSup.deleteMark, GET_TASKID(pTaskInfo), pHandle->checkpointId, STREAM_STATE_BUFF_HASH,

×

5746

      &pInfo->pState->pFileState);

×

5747

  QUERY_CHECK_CODE(code, lino, _error);

×

5749

  pInfo->pOperator = pOperator;

×

5750

  setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED,

×

5753

      createOperatorFpSet(optrDummyOpenFn, doStreamIntervalAggNext, NULL, destroyStreamFinalIntervalOperatorInfo,

×

5755

  setOperatorStreamStateFn(pOperator, streamIntervalReleaseState, streamIntervalReloadState);

×

5757

  code = initStreamBasicInfo(&pInfo->basic, pOperator);

×

5758

  QUERY_CHECK_CODE(code, lino, _error);

×

5760

  pInfo->recvGetAll = false;

×

5762

  code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);

×

5763

  QUERY_CHECK_CODE(code, lino, _error);

×

5765

  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

5766

  pInfo->pDeletedMap = tSimpleHashInit(4096, hashFn);

×

5767

  QUERY_CHECK_NULL(pInfo->pDeletedMap, code, lino, _error, terrno);

×

5768

  pInfo->destHasPrimaryKey = pIntervalPhyNode->window.destHasPrimaryKey;

×

5771

  void*   buff = NULL;

×

5772

  int32_t len = 0;

×

5773

  int32_t res = pAPI->stateStore.streamStateGetInfo(pInfo->pState, STREAM_INTERVAL_OP_CHECKPOINT_NAME,

×

5775

  if (res == TSDB_CODE_SUCCESS) {

×

5776

    doStreamIntervalDecodeOpState(buff, len, pOperator);

×

5777

    taosMemoryFree(buff);

×

5780

  pInfo->basic.primaryPkIndex = -1;

×

5781

  code = initIntervalDownStream(downstream, pPhyNode->type, pInfo, &pInfo->basic);

×

5782

  QUERY_CHECK_CODE(code, lino, _error);

×

5784

  code = appendDownstream(pOperator, &downstream, 1);

×

5785

  QUERY_CHECK_CODE(code, lino, _error);

×

5787

  *pOptrInfo = pOperator;

×

5788

  return TSDB_CODE_SUCCESS;

×

5790

_error:

×

5791

  if (pInfo != NULL) destroyStreamFinalIntervalOperatorInfo(pInfo);

×

5792

  destroyOperatorAndDownstreams(pOperator, &downstream, 1);

×

5793

  pTaskInfo->code = code;

×

5794

  return code;

×

5797

int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo,

×

5799

  SStreamIntervalPhysiNode* pIntervalPhyNode = (SStreamIntervalPhysiNode*)pPhyNode;

×

5800

  if (pIntervalPhyNode->window.triggerType == STREAM_TRIGGER_FORCE_WINDOW_CLOSE) {

×

5801

    return createStreamIntervalSliceOperatorInfo(downstream, pPhyNode, pTaskInfo, pHandle, pOptrInfo);

×

5803

    return createStreamSingleIntervalOperatorInfo(downstream, pPhyNode, pTaskInfo, pHandle, pOptrInfo);

×

5808

static void doStreamMidIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pUpdatedMap) {

×

5809

  int32_t                      code = TSDB_CODE_SUCCESS;

×

5810

  int32_t                      lino = 0;

×

5811

  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperator->info;

×

5812

  pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version);

×

5814

  SResultRowInfo*  pResultRowInfo = &(pInfo->binfo.resultRowInfo);

×

5815

  SExecTaskInfo*   pTaskInfo = pOperator->pTaskInfo;

×

5816

  SExprSupp*       pSup = &pOperator->exprSupp;

×

5817

  int32_t          numOfOutput = pSup->numOfExprs;

×

5818

  int32_t          step = 1;

×

5819

  SRowBuffPos*     pResPos = NULL;

×

5820

  SResultRow*      pResult = NULL;

×

5821

  int32_t          forwardRows = 1;

×

5822

  uint64_t         groupId = pSDataBlock->info.id.groupId;

×

5823

  SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);

×

5824

  TSKEY*           tsCol = (int64_t*)pColDataInfo->pData;

×

5826

  int32_t     startPos = 0;

×

5827

  TSKEY       ts = getStartTsKey(&pSDataBlock->info.window, tsCol);

×

5828

  STimeWindow nextWin = getFinalTimeWindow(ts, &pInfo->interval);

×

5830

  while (1) {

×

5831

    SWinKey key = {

×

5832

        .ts = nextWin.skey,

×

5835

    void*   chIds = taosHashGet(pInfo->pPullDataMap, &key, sizeof(SWinKey));

×

5836

    int32_t index = -1;

×

5837

    SArray* chArray = NULL;

×

5838

    int32_t chId = 0;

×

5839

    if (chIds) {

×

5840

      chArray = *(void**)chIds;

×

5841

      chId = getChildIndex(pSDataBlock);

×

5842

      index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ);

×

5844

    if (!(index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA)) {

×

5845

      startPos = getNextQualifiedFinalWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCol, startPos);

×

5846

      if (startPos < 0) {

×

5847

        break;

×

5849

      continue;

×

5852

    if (!inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) {

×

5853

      startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCol, startPos, nextWin.ekey, &nextWin);

×

5854

      if (startPos < 0) {

×

5855

        break;

×

5857

      continue;

×

5860

    int32_t winCode = TSDB_CODE_SUCCESS;

×

5861

    code = setIntervalOutputBuf(pInfo->pState, &nextWin, &pResPos, groupId, pSup->pCtx, numOfOutput,

×

5863

    QUERY_CHECK_CODE(code, lino, _end);

×

5865

    pResult = (SResultRow*)pResPos->pRowBuff;

×

5867

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {

×

5868

      code = saveWinResult(&key, pResPos, pUpdatedMap);

×

5869

      QUERY_CHECK_CODE(code, lino, _end);

×

5872

    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {

×

5873

      code = tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pResPos, POINTER_BYTES);

×

5874

      QUERY_CHECK_CODE(code, lino, _end);

×

5877

    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, 1);

×

5878

    code = applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos,

×

5879

                                           forwardRows, pSDataBlock->info.rows, numOfOutput);

×

5880

    QUERY_CHECK_CODE(code, lino, _end);

×

5881

    key.ts = nextWin.skey;

×

5883

    if (pInfo->delKey.ts > key.ts) {

×

5884

      pInfo->delKey = key;

×

5886

    int32_t prevEndPos = (forwardRows - 1) * step + startPos;

×

5887

    if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) {

×

5888

      qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64

×

5891

      code = blockDataUpdateTsWindow(pSDataBlock, 0);

×

5892

      QUERY_CHECK_CODE(code, lino, _end);

×

5895

      if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) {

×

5896

        qError("table uid %" PRIu64 " data block timestamp is out of range! minKey %" PRId64 ",maxKey %" PRId64,

×

5900

    startPos = getNextQualifiedFinalWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCol, prevEndPos);

×

5901

    if (startPos < 0) {

×

5902

      break;

×

5906

_end:

×

5907

  if (code != TSDB_CODE_SUCCESS) {

×

5908

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

5910

×

5912

static int32_t addMidRetriveWindow(SArray* wins, SHashObj* pMidPullMap, int32_t numOfChild) {

×

5913

  int32_t code = TSDB_CODE_SUCCESS;

×

5914

  int32_t lino = 0;

×

5915

  int32_t size = taosArrayGetSize(wins);

×

5916

  for (int32_t i = 0; i < size; i++) {

×

5917

    SWinKey* winKey = taosArrayGet(wins, i);

×

5918

    void*    chIds = taosHashGet(pMidPullMap, winKey, sizeof(SWinKey));

×

5919

    if (!chIds) {

×

5920

      code = addPullWindow(pMidPullMap, winKey, numOfChild);

×

5921

      qDebug("===stream===prepare mid operator retrive for delete %" PRId64 ", size:%d", winKey->ts, numOfChild);

×

5922

      QUERY_CHECK_CODE(code, lino, _end);

×

5925

_end:

×

5926

  if (code != TSDB_CODE_SUCCESS) {

×

5927

    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));

×

5929

  return code;

×

5932

static SSDataBlock* buildMidIntervalResult(SOperatorInfo* pOperator) {

×

5933

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

5934

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

5935

  uint16_t                     opType = pOperator->operatorType;

×

5937

  if (pInfo->recvPullover) {

×

5938

    pInfo->recvPullover = false;

×

5939

    printDataBlock(pInfo->pMidPulloverRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5940

    return pInfo->pMidPulloverRes;

×

5943

  qDebug("===stream=== build mid interval result");

×

5944

  doBuildDeleteResult(pInfo, pInfo->pMidPullDatas, &pInfo->midDelIndex, pInfo->pDelRes);

×

5945

  if (pInfo->pDelRes->info.rows != 0) {

×

5947

    printDataBlock(pInfo->pDelRes, getStreamOpName(opType), GET_TASKID(pTaskInfo));

×

5948

    return pInfo->pDelRes;

×

5951

  if (pInfo->recvRetrive) {

×

5952

    pInfo->recvRetrive = false;

×

5953

    printDataBlock(pInfo->pMidRetriveRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

5954

    return pInfo->pMidRetriveRes;

×

5957

  return NULL;

×

5960

static int32_t doStreamMidIntervalAggNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {

×

5961

  int32_t                      code = TSDB_CODE_SUCCESS;

×

5962

  int32_t                      lino = 0;

×

5963

  SStreamIntervalOperatorInfo* pInfo = pOperator->info;

×

5964

  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;

×

5965

  SStorageAPI*                 pAPI = &pOperator->pTaskInfo->storageAPI;

×

5966

  SOperatorInfo*               downstream = pOperator->pDownstream[0];

×

5967

  SExprSupp*                   pSup = &pOperator->exprSupp;

×

5969

  qDebug("stask:%s  %s status: %d", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status);

×

5971

  if (pOperator->status == OP_EXEC_DONE) {

×

5972

    (*ppRes) = NULL;

×

5973

    return code;

×

5974

  } else if (pOperator->status == OP_RES_TO_RETURN) {

×

5975

    SSDataBlock* resBlock = NULL;

×

5976

    code = buildIntervalResult(pOperator, &resBlock);

×

5977

    QUERY_CHECK_CODE(code, lino, _end);

×

5978

    if (resBlock != NULL) {

×

5979

      (*ppRes) = resBlock;

×

5980

      return code;

×

5983

    setOperatorCompleted(pOperator);

×

5984

    clearFunctionContext(&pOperator->exprSupp);

×

5985

    clearStreamIntervalOperator(pInfo);

×

5986

    qDebug("stask:%s  ===stream===%s clear", GET_TASKID(pTaskInfo), getStreamOpName(pOperator->operatorType));

×

5987

    (*ppRes) = NULL;

×

5988

    return code;

×

5990

    SSDataBlock* resBlock = NULL;

×

5991

    code = buildIntervalResult(pOperator, &resBlock);

×

5992

    QUERY_CHECK_CODE(code, lino, _end);

×

5993

    if (resBlock != NULL) {

×

5994

      (*ppRes) = resBlock;

×

5995

      return code;

×

5998

    resBlock = buildMidIntervalResult(pOperator);

×

5999

    if (resBlock != NULL) {

×

6000

      (*ppRes) = resBlock;

×

6001

      return code;

×

6004

    if (pInfo->clearState) {

×

6005

      pInfo->clearState = false;

×

6006

      clearFunctionContext(&pOperator->exprSupp);

×

6007

      clearStreamIntervalOperator(pInfo);

×

6011

  if (!pInfo->pUpdated) {

×

6012

    pInfo->pUpdated = taosArrayInit(4096, POINTER_BYTES);

×

6013

    QUERY_CHECK_NULL(pInfo->pUpdated, code, lino, _end, terrno);

×

6015

  if (!pInfo->pUpdatedMap) {

×

6016

    _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);

×

6017

    pInfo->pUpdatedMap = tSimpleHashInit(4096, hashFn);

×

6018

    QUERY_CHECK_NULL(pInfo->pUpdatedMap, code, lino, _end, terrno);

×

6021

  while (1) {

×

6022

    if (isTaskKilled(pTaskInfo)) {

×

6023

      qInfo("===stream=== %s task is killed, code %s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code));

×

6024

      (*ppRes) = NULL;

×

6025

      return code;

×

6028

    SSDataBlock* pBlock = NULL;

×

6029

    code = downstream->fpSet.getNextFn(downstream, &pBlock);

×

6030

    QUERY_CHECK_CODE(code, lino, _end);

×

6032

    if (pBlock == NULL) {

×

6033

      pOperator->status = OP_RES_TO_RETURN;

×

6034

      qDebug("===stream===return data:%s. recv datablock num:%" PRIu64, getStreamOpName(pOperator->operatorType),

×

6036

      pInfo->numOfDatapack = 0;

×

6037

      break;

×

6039

    pInfo->numOfDatapack++;

×

6040

    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo));

×

6041

    setStreamOperatorState(&pInfo->basic, pBlock->info.type);

×

6043

    if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) {

×

6044

      pInfo->binfo.pRes->info.type = pBlock->info.type;

×

6045

    } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT ||

×

6046

               pBlock->info.type == STREAM_CLEAR) {

×

6047

      SArray* delWins = taosArrayInit(8, sizeof(SWinKey));

×

6048

      if (!delWins) {

×

6049

        code = terrno;

×

6050

        QUERY_CHECK_CODE(code, lino, _end);

×

6053

          doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, pInfo->pFinalPullDataMap);

×

6054

      QUERY_CHECK_CODE(code, lino, _end);

×

6056

      removeResults(delWins, pInfo->pUpdatedMap);

×

6057

      void* tmp = taosArrayAddAll(pInfo->pDelWins, delWins);

×

6058

      if (!tmp && taosArrayGetSize(delWins) > 0) {

×

6059

        code = TSDB_CODE_OUT_OF_MEMORY;

×

6060

        QUERY_CHECK_CODE(code, lino, _end);

×

6062

      taosArrayDestroy(delWins);

×

6064

      doBuildDeleteResult(pInfo, pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);

×

6065

      if (pInfo->pDelRes->info.rows != 0) {

×

6067

        printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));

×

6068

        if (pBlock->info.type == STREAM_CLEAR) {

×

6069

          pInfo->pDelRes->info.type = STREAM_CLEAR;

×

6071

          pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;

×

6073

        (*ppRes) = pInfo->pDelRes;

×

6074

        return code;

×

6076

      continue;

×

6077

    } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) {

×

6078

      (*ppRes) = pBlock;

×

6079

      return code;

×

6080

    } else if (pBlock->info.type == STREAM_PULL_OVER) {

×

6081

      code = processPullOver(pBlock, pInfo->pPullDataMap, pInfo->pFinalPullDataMap, &pInfo->interval, pInfo->pPullWins,

×

6083

      QUERY_CHECK_CODE(code, lino, _end);

×

6085

      if (pInfo->recvPullover) {

×

6086

        code = copyDataBlock(pInfo->pMidPulloverRes, pBlock);

×

6087

        QUERY_CHECK_CODE(code, lino, _end);

×

6089

        pInfo->clearState = true;

×

6090

        break;

×

6092

      continue;

×

6093

    } else if (pBlock->info.type == STREAM_CHECKPOINT) {

×

6094

      pAPI->stateStore.streamStateCommit(pInfo->pState);

×

6095

      doStreamIntervalSaveCheckpoint(pOperator);

×

6096

      code = copyDataBlock(pInfo->pCheckpointRes, pBlock);

×

6097

      QUERY_CHECK_CODE(code, lino, _end);

×

6099

      continue;

×

6100

    } else if (pBlock->info.type == STREAM_MID_RETRIEVE) {

×

6101

      SArray* delWins = taosArrayInit(8, sizeof(SWinKey));

×

6102

      if (!delWins) {

×

6103

        code = terrno;

×

6104

        QUERY_CHECK_CODE(code, lino, _end);

×

6106

      code = doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap, NULL);

×

6107

      QUERY_CHECK_CODE(code, lino, _end);

×

6109

      code = addMidRetriveWindow(delWins, pInfo->pPullDataMap, pInfo->numOfChild);

×

6110

      QUERY_CHECK_CODE(code, lino, _end);

×

6112

      taosArrayDestroy(delWins);

×

6113

      pInfo->recvRetrive = true;

×

6114

      code = copyDataBlock(pInfo->pMidRetriveRes, pBlock);

×

6115

      QUERY_CHECK_CODE(code, lino, _end);

×

6117

      pInfo->pMidRetriveRes->info.type = STREAM_MID_RETRIEVE;

×

6118

      pInfo->clearState = true;

×

6119

      break;

×

6121

      if (pBlock->info.type != STREAM_INVALID) {

×

6122

        code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;

×

6123

        QUERY_CHECK_CODE(code, lino, _end);

×

6127

    if (pInfo->scalarSupp.pExprInfo != NULL) {

×

6128

      SExprSupp* pExprSup = &pInfo->scalarSupp;

×

6129

      code = projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);

×

6130

      QUERY_CHECK_CODE(code, lino, _end);

×

6132

    code = setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);

×

6133

    QUERY_CHECK_CODE(code, lino, _end);

×

6134

    doStreamMidIntervalAggImpl(pOperator, pBlock, pInfo->pUpdatedMap);

×

6135

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);

×

6136

    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.watermark);

×

6137

    pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey);

×

6140

  removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins);

×

6141

  pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs;

×

6143

  void*   pIte = NULL;

×

6144

  int32_t iter = 0;

×

6145

  while ((pIte = tSimpleHashIterate(pInfo->pUpdatedMap, pIte, &iter)) != NULL) {

×

6146

    void* tmp = taosArrayPush(pInfo->pUpdated, pIte);

×

6147

    if (!tmp) {

×

6148

      code = terrno;

×

6149

      QUERY_CHECK_CODE(code, lino, _end);

×

6153

  tSimpleHashCleanup(pInfo->pUpdatedMap);

×

6154

  pInfo->pUpdatedMap = NULL;

×

6155

  taosArraySort(pInfo->pUpdated, winPosCmprImpl);

×

6157

  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated);

×

6158

  pInfo->pUpdated = NULL;

×

6159

  code = blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);

×

6160

  QUERY_CHECK_CODE(code, lino, _end);

×

6162

  SSDataBlock* resBlock = NULL;

×

6163

  code = buildIntervalResult(pOperator, &resBlock);

×

6164

  QUERY_CHECK_CODE(code, lino, _end);

×

6165

  if (resBlock != NULL) {

×

6166

    (*ppRes) = resBlock;

×

6167

    return code;

×

6170

  resBlock = buildMidIntervalResult(pOperator);

×

6171

  if (resBlock != NULL) {

×

6172

    (*ppRes) = resBlock;

×

6173

    return code;

×

6176

  if (pInfo->clearState) {

×

6177

    pInfo->clearState = false;

×

6178

    clearFunctionContext(&pOperator->exprSupp);

×

6179

    clearStreamIntervalOperator(pInfo);

×

6182

_end:

×

6183

  if (code != TSDB_CODE_SUCCESS) {

×

6184

    qError("%s failed at line %d since %s. task:%s", __func__, lino, tstrerror(code), GET_TASKID(pTaskInfo));

×

6185

    pTaskInfo->code = code;

×

6186

    T_LONG_JMP(pTaskInfo->env, code);

×

6188

  (*ppRes) = NULL;

×

6189

  return code;

×

6192

static SSDataBlock* doStreamMidIntervalAgg(SOperatorInfo* pOperator) {

×

6193

  SSDataBlock* pRes = NULL;

×

6194

  int32_t      code = doStreamMidIntervalAggNext(pOperator, &pRes);

×

6195

  return pRes;

×

6198

void setStreamOperatorCompleted(SOperatorInfo* pOperator) {

×

6199

  qDebug("stask:%s  %s status: %d. set completed", GET_TASKID(pOperator->pTaskInfo),

×

6201

  setOperatorCompleted(pOperator);

×

6202

×

taosdata / TDengine / #3638

Source File
Press 'n' to go to next uncovered line, 'b' for previous

taosdata / TDengine / #3638

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous