#4808

Committed 16 Oct 2025 11:40AM UTC coverage: 57.938% (-0.6%) from 58.524%

Build # #4808

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

fix(tref): increase TSDB_REF_OBJECTS from 100 to 2000 for improved reference handling (#33281)

Run Details

137662 of 303532 branches covered (45.35%)

Branch coverage included in aggregate %.

209234 of 295200 relevant lines covered (70.88%)

4035326.15 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

43.52

/source/dnode/vnode/src/meta/metaTtl.c

/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#include "metaTtl.h"
#include "meta.h"
#include "tdbInt.h"

typedef struct {
  TTB   *pNewTtlIdx;
  SMeta *pMeta;
} SConvertData;

typedef struct {
  int32_t      ttlDropMaxCount;
  int32_t      count;
  STtlIdxKeyV1 expiredKey;
  SArray      *pTbUids;
} STtlExpiredCtx;

static void ttlMgrCleanup(STtlManger *pTtlMgr);

static int ttlMgrConvert(TTB *pOldTtlIdx, TTB *pNewTtlIdx, void *pMeta);

static void    ttlMgrBuildKey(STtlIdxKeyV1 *pTtlKey, int64_t ttlDays, int64_t changeTimeMs, tb_uid_t uid);
static int     ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
static int     ttlIdxKeyV1Cmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2);
static int     ttlMgrFillCache(STtlManger *pTtlMgr);
static int32_t ttlMgrFillCacheOneEntry(const void *pKey, int keyLen, const void *pVal, int valLen, void *pTtlCache);
static int32_t ttlMgrConvertOneEntry(const void *pKey, int keyLen, const void *pVal, int valLen, void *pConvertData);
static int32_t ttlMgrFindExpiredOneEntry(const void *pKey, int keyLen, const void *pVal, int valLen,
                                         void *pExpiredInfo);

static bool ttlMgrNeedFlush(STtlManger *pTtlMgr);

const char *ttlTbname = "ttl.idx";
const char *ttlV1Tbname = "ttlv1.idx";

int32_t ttlMgrOpen(STtlManger **ppTtlMgr, TDB *pEnv, int8_t rollback, const char *logPrefix, int32_t flushThreshold) {
  int32_t code = TSDB_CODE_SUCCESS;
  int64_t startNs = taosGetTimestampNs();
  int32_t pathLen = 0;

  *ppTtlMgr = NULL;

  STtlManger *pTtlMgr = (STtlManger *)tdbOsCalloc(1, sizeof(*pTtlMgr));
  if (pTtlMgr == NULL) TAOS_RETURN(terrno);

  pathLen = strlen(logPrefix) + 1;
  char *logBuffer = (char *)tdbOsCalloc(1, pathLen);
  if (logBuffer == NULL) {
    tdbOsFree(pTtlMgr);
    TAOS_RETURN(terrno);
  }
  tstrncpy(logBuffer, logPrefix, pathLen);
  pTtlMgr->logPrefix = logBuffer;
  pTtlMgr->flushThreshold = flushThreshold;

  code = tdbTbOpen(ttlV1Tbname, TDB_VARIANT_LEN, TDB_VARIANT_LEN, ttlIdxKeyV1Cmpr, pEnv, &pTtlMgr->pTtlIdx, rollback);
  if (TSDB_CODE_SUCCESS != code) {
    metaError("%s, failed to open %s since %s", pTtlMgr->logPrefix, ttlV1Tbname, tstrerror(code));
    tdbOsFree(pTtlMgr);
    TAOS_RETURN(code);
  }

  pTtlMgr->pTtlCache = taosHashInit(8192, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
  pTtlMgr->pDirtyUids = taosHashInit(8192, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);

  if ((code = ttlMgrFillCache(pTtlMgr)) != TSDB_CODE_SUCCESS) {
    metaError("%s, failed to fill hash since %s", pTtlMgr->logPrefix, tstrerror(terrno));
    ttlMgrCleanup(pTtlMgr);
    TAOS_RETURN(code);
  }

  int64_t endNs = taosGetTimestampNs();
  metaInfo("%s, ttl mgr open end, hash size: %d, time consumed: %" PRId64 " ns", pTtlMgr->logPrefix,
           taosHashGetSize(pTtlMgr->pTtlCache), endNs - startNs);

  *ppTtlMgr = pTtlMgr;
  TAOS_RETURN(TSDB_CODE_SUCCESS);
}

void ttlMgrClose(STtlManger *pTtlMgr) { ttlMgrCleanup(pTtlMgr); }

bool ttlMgrNeedUpgrade(TDB *pEnv) {
  bool needUpgrade = tdbTbExist(ttlTbname, pEnv);
  if (needUpgrade) {
    metaInfo("find ttl idx in old version , will convert");
  }
  return needUpgrade;
}

int32_t ttlMgrUpgrade(STtlManger *pTtlMgr, void *pMeta) {
  SMeta  *meta = (SMeta *)pMeta;
  int32_t code = TSDB_CODE_SUCCESS;

  if (!tdbTbExist(ttlTbname, meta->pEnv)) TAOS_RETURN(TSDB_CODE_SUCCESS);

  metaInfo("%s, ttl mgr start upgrade", pTtlMgr->logPrefix);

  int64_t startNs = taosGetTimestampNs();

  code = tdbTbOpen(ttlTbname, sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, meta->pEnv, &pTtlMgr->pOldTtlIdx, 0);
  if (TSDB_CODE_SUCCESS != code) {
    metaError("%s, failed to open %s index since %s", pTtlMgr->logPrefix, ttlTbname, tstrerror(code));
    goto _out;
  }

  if ((code = ttlMgrConvert(pTtlMgr->pOldTtlIdx, pTtlMgr->pTtlIdx, pMeta)) != TSDB_CODE_SUCCESS) {
    metaError("%s, failed to convert ttl index since %s", pTtlMgr->logPrefix, tstrerror(code));
    goto _out;
  }

  if ((code = tdbTbDropByName(ttlTbname, meta->pEnv, meta->txn)) != TSDB_CODE_SUCCESS) {
    metaError("%s, failed to drop old ttl index since %s", pTtlMgr->logPrefix, tstrerror(code));
    goto _out;
  }

  if ((code = ttlMgrFillCache(pTtlMgr)) != TSDB_CODE_SUCCESS) {
    metaError("%s, failed to fill hash since %s", pTtlMgr->logPrefix, tstrerror(code));
    goto _out;
  }

  int64_t endNs = taosGetTimestampNs();
  metaInfo("%s, ttl mgr upgrade end, hash size: %d, time consumed: %" PRId64 " ns", pTtlMgr->logPrefix,
           taosHashGetSize(pTtlMgr->pTtlCache), endNs - startNs);

_out:
  tdbTbClose(pTtlMgr->pOldTtlIdx);
  pTtlMgr->pOldTtlIdx = NULL;

  TAOS_RETURN(code);
}

static void ttlMgrCleanup(STtlManger *pTtlMgr) {
  taosMemoryFree(pTtlMgr->logPrefix);
  taosHashCleanup(pTtlMgr->pTtlCache);
  taosHashCleanup(pTtlMgr->pDirtyUids);
  tdbTbClose(pTtlMgr->pTtlIdx);
  taosMemoryFree(pTtlMgr);
}

static void ttlMgrBuildKey(STtlIdxKeyV1 *pTtlKey, int64_t ttlDays, int64_t changeTimeMs, tb_uid_t uid) {
  if (ttlDays <= 0) return;

  pTtlKey->deleteTimeMs = changeTimeMs + ttlDays * tsTtlUnit * 1000;
  pTtlKey->uid = uid;
}

static int ttlIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
  STtlIdxKey *pTtlIdxKey1 = (STtlIdxKey *)pKey1;
  STtlIdxKey *pTtlIdxKey2 = (STtlIdxKey *)pKey2;

  if (pTtlIdxKey1->deleteTimeSec > pTtlIdxKey2->deleteTimeSec) {
    return 1;
  } else if (pTtlIdxKey1->deleteTimeSec < pTtlIdxKey2->deleteTimeSec) {
    return -1;
  }

  if (pTtlIdxKey1->uid > pTtlIdxKey2->uid) {
    return 1;
  } else if (pTtlIdxKey1->uid < pTtlIdxKey2->uid) {
    return -1;
  }

  return 0;
}

static int ttlIdxKeyV1Cmpr(const void *pKey1, int kLen1, const void *pKey2, int kLen2) {
  STtlIdxKeyV1 *pTtlIdxKey1 = (STtlIdxKeyV1 *)pKey1;
  STtlIdxKeyV1 *pTtlIdxKey2 = (STtlIdxKeyV1 *)pKey2;

  if (pTtlIdxKey1->deleteTimeMs > pTtlIdxKey2->deleteTimeMs) {
    return 1;
  } else if (pTtlIdxKey1->deleteTimeMs < pTtlIdxKey2->deleteTimeMs) {
    return -1;
  }

  if (pTtlIdxKey1->uid > pTtlIdxKey2->uid) {
    return 1;
  } else if (pTtlIdxKey1->uid < pTtlIdxKey2->uid) {
    return -1;
  }

  return 0;
}

static int ttlMgrFillCache(STtlManger *pTtlMgr) {
  return tdbTbTraversal(pTtlMgr->pTtlIdx, pTtlMgr->pTtlCache, ttlMgrFillCacheOneEntry);
}

static int32_t ttlMgrFillCacheOneEntry(const void *pKey, int keyLen, const void *pVal, int valLen, void *pTtlCache) {
  SHashObj *pCache = (SHashObj *)pTtlCache;

  STtlIdxKeyV1 *ttlKey = (STtlIdxKeyV1 *)pKey;
  tb_uid_t      uid = ttlKey->uid;
  int64_t       ttlDays = *(int64_t *)pVal;
  int64_t       changeTimeMs = ttlKey->deleteTimeMs - ttlDays * tsTtlUnit * 1000;

  STtlCacheEntry data = {
      .ttlDays = ttlDays, .changeTimeMs = changeTimeMs, .ttlDaysDirty = ttlDays, .changeTimeMsDirty = changeTimeMs};

  return taosHashPut(pCache, &uid, sizeof(uid), &data, sizeof(data));
}

static int32_t ttlMgrConvertOneEntry(const void *pKey, int keyLen, const void *pVal, int valLen, void *pConvertData) {
  SConvertData *pData = (SConvertData *)pConvertData;

  STtlIdxKey *ttlKey = (STtlIdxKey *)pKey;
  tb_uid_t    uid = ttlKey->uid;
  int64_t     ttlDays = 0;

  int32_t code = TSDB_CODE_SUCCESS;
  if ((code = metaGetTableTtlByUid(pData->pMeta, uid, &ttlDays)) != TSDB_CODE_SUCCESS) {
    metaError("ttlMgr convert failed to get ttl since %s", tstrerror(code));
    goto _out;
  }

  STtlIdxKeyV1 ttlKeyV1 = {.deleteTimeMs = ttlKey->deleteTimeSec * 1000, .uid = uid};
  code = tdbTbUpsert(pData->pNewTtlIdx, &ttlKeyV1, sizeof(ttlKeyV1), &ttlDays, sizeof(ttlDays), pData->pMeta->txn);
  if (code != TSDB_CODE_SUCCESS) {
    metaError("ttlMgr convert failed to upsert since %s", tstrerror(code));
    goto _out;
  }

  code = TSDB_CODE_SUCCESS;

_out:
  TAOS_RETURN(code);
}

static int32_t ttlMgrFindExpiredOneEntry(const void *pKey, int keyLen, const void *pVal, int valLen,
                                         void *pExpiredCtx) {
  STtlExpiredCtx *pCtx = (STtlExpiredCtx *)pExpiredCtx;
  if (pCtx->count >= pCtx->ttlDropMaxCount) return -1;

  int c = ttlIdxKeyV1Cmpr(&pCtx->expiredKey, sizeof(pCtx->expiredKey), pKey, keyLen);
  if (c > 0) {
    if (NULL == taosArrayPush(pCtx->pTbUids, &((STtlIdxKeyV1 *)pKey)->uid)) {
      metaError("ttlMgr find expired failed since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY));
      return -1;
    }
    pCtx->count++;
  }

  return c;
}

// static int32_t ttlMgrDumpOneEntry(const void *pKey, int keyLen, const void *pVal, int valLen, void *pDumpCtx) {
//   STtlIdxKeyV1 *ttlKey = (STtlIdxKeyV1 *)pKey;
//   int64_t      *ttlDays = (int64_t *)pVal;

//   metaInfo("ttlMgr dump, ttl: %" PRId64 ", ctime: %" PRId64 ", uid: %" PRId64, *ttlDays, ttlKey->deleteTimeMs,
//            ttlKey->uid);

//   TAOS_RETURN(TSDB_CODE_SUCCESS);
// }

static int ttlMgrConvert(TTB *pOldTtlIdx, TTB *pNewTtlIdx, void *pMeta) {
  SMeta *meta = pMeta;

  metaInfo("ttlMgr convert start.");

  SConvertData cvData = {.pNewTtlIdx = pNewTtlIdx, .pMeta = meta};

  int code = TSDB_CODE_SUCCESS;
  if ((code = tdbTbTraversal(pOldTtlIdx, &cvData, ttlMgrConvertOneEntry)) != TSDB_CODE_SUCCESS) {
    metaError("failed to convert since %s", tstrerror(code));
  }

  metaInfo("ttlMgr convert end.");
  TAOS_RETURN(code);
}

int32_t ttlMgrInsertTtl(STtlManger *pTtlMgr, const STtlUpdTtlCtx *updCtx) {
  if (updCtx->ttlDays == 0) return 0;

  STtlCacheEntry cacheEntry = {.ttlDays = updCtx->ttlDays,
                               .changeTimeMs = updCtx->changeTimeMs,
                               .ttlDaysDirty = updCtx->ttlDays,
                               .changeTimeMsDirty = updCtx->changeTimeMs};
  STtlDirtyEntry dirtryEntry = {.type = ENTRY_TYPE_UPSERT};

  int32_t code = taosHashPut(pTtlMgr->pTtlCache, &updCtx->uid, sizeof(updCtx->uid), &cacheEntry, sizeof(cacheEntry));
  if (TSDB_CODE_SUCCESS != code) {
    metaError("%s, ttlMgr insert failed to update cache since %s", pTtlMgr->logPrefix, tstrerror(code));
    goto _out;
  }

  code = taosHashPut(pTtlMgr->pDirtyUids, &updCtx->uid, sizeof(updCtx->uid), &dirtryEntry, sizeof(dirtryEntry));
  if (TSDB_CODE_SUCCESS != code) {
    metaError("%s, ttlMgr insert failed to update dirty uids since %s", pTtlMgr->logPrefix, tstrerror(code));
    goto _out;
  }

  if (ttlMgrNeedFlush(pTtlMgr)) {
    int32_t ret = ttlMgrFlush(pTtlMgr, updCtx->pTxn);
    if (ret < 0) {
      metaError("%s, ttlMgr insert failed to flush since %s", pTtlMgr->logPrefix, tstrerror(ret));
    }
  }

  code = TSDB_CODE_SUCCESS;

_out:
  metaTrace("%s, ttl mgr insert ttl, uid: %" PRId64 ", ctime: %" PRId64 ", ttlDays: %" PRId64, pTtlMgr->logPrefix,
            updCtx->uid, updCtx->changeTimeMs, updCtx->ttlDays);

  TAOS_RETURN(code);
}

int32_t ttlMgrDeleteTtl(STtlManger *pTtlMgr, const STtlDelTtlCtx *delCtx) {
  if (delCtx->ttlDays == 0) return 0;

  STtlDirtyEntry dirtryEntry = {.type = ENTRY_TYPE_DELETE};

  int32_t code = taosHashPut(pTtlMgr->pDirtyUids, &delCtx->uid, sizeof(delCtx->uid), &dirtryEntry, sizeof(dirtryEntry));
  if (TSDB_CODE_SUCCESS != code) {
    metaError("%s, ttlMgr del failed to update dirty uids since %s", pTtlMgr->logPrefix, tstrerror(code));
    goto _out;
  }

  if (ttlMgrNeedFlush(pTtlMgr)) {
    int32_t ret = ttlMgrFlush(pTtlMgr, delCtx->pTxn);
    if (ret < 0) {
      metaError("%s, ttlMgr del failed to flush since %s", pTtlMgr->logPrefix, tstrerror(ret));
    }
  }

  code = TSDB_CODE_SUCCESS;

_out:
  metaTrace("%s, ttl mgr delete ttl, uid: %" PRId64, pTtlMgr->logPrefix, delCtx->uid);
  TAOS_RETURN(code);
}

int32_t ttlMgrUpdateChangeTime(STtlManger *pTtlMgr, const STtlUpdCtimeCtx *pUpdCtimeCtx) {
  int32_t code = TSDB_CODE_SUCCESS;

  STtlCacheEntry *oldData = taosHashGet(pTtlMgr->pTtlCache, &pUpdCtimeCtx->uid, sizeof(pUpdCtimeCtx->uid));
  if (oldData == NULL) {
    goto _out;
  }

  STtlCacheEntry cacheEntry = {.ttlDays = oldData->ttlDays,
                               .changeTimeMs = oldData->changeTimeMs,
                               .ttlDaysDirty = oldData->ttlDays,
                               .changeTimeMsDirty = pUpdCtimeCtx->changeTimeMs};
  STtlDirtyEntry dirtryEntry = {.type = ENTRY_TYPE_UPSERT};

  code =
      taosHashPut(pTtlMgr->pTtlCache, &pUpdCtimeCtx->uid, sizeof(pUpdCtimeCtx->uid), &cacheEntry, sizeof(cacheEntry));
  if (TSDB_CODE_SUCCESS != code) {
    metaError("%s, ttlMgr update ctime failed to update cache since %s", pTtlMgr->logPrefix, tstrerror(code));
    goto _out;
  }

  code = taosHashPut(pTtlMgr->pDirtyUids, &pUpdCtimeCtx->uid, sizeof(pUpdCtimeCtx->uid), &dirtryEntry,
                     sizeof(dirtryEntry));
  if (TSDB_CODE_SUCCESS != code) {
    metaError("%s, ttlMgr update ctime failed to update dirty uids since %s", pTtlMgr->logPrefix, tstrerror(code));
    goto _out;
  }

  if (ttlMgrNeedFlush(pTtlMgr)) {
    int32_t ret = ttlMgrFlush(pTtlMgr, pUpdCtimeCtx->pTxn);
    if (ret < 0) {
      metaError("%s, ttlMgr update ctime failed to flush since %s", pTtlMgr->logPrefix, tstrerror(ret));
    }
  }

  code = TSDB_CODE_SUCCESS;

_out:
  metaTrace("%s, ttl mgr update ctime, uid: %" PRId64 ", ctime: %" PRId64, pTtlMgr->logPrefix, pUpdCtimeCtx->uid,
            pUpdCtimeCtx->changeTimeMs);
  TAOS_RETURN(code);
}

int32_t ttlMgrFindExpired(STtlManger *pTtlMgr, int64_t timePointMs, SArray *pTbUids, int32_t ttlDropMaxCount) {
  int32_t code = TSDB_CODE_SUCCESS;

  STtlIdxKeyV1   ttlKey = {.deleteTimeMs = timePointMs, .uid = INT64_MAX};
  STtlExpiredCtx expiredCtx = {
      .ttlDropMaxCount = ttlDropMaxCount, .count = 0, .expiredKey = ttlKey, .pTbUids = pTbUids};
  TAOS_CHECK_GOTO(tdbTbTraversal(pTtlMgr->pTtlIdx, &expiredCtx, ttlMgrFindExpiredOneEntry), NULL, _out);

  size_t vIdx = 0;
  for (size_t i = 0; i < pTbUids->size; i++) {
    tb_uid_t *pUid = taosArrayGet(pTbUids, i);
    if (taosHashGet(pTtlMgr->pDirtyUids, pUid, sizeof(tb_uid_t)) == NULL) {
      // not in dirty && expired in tdb => must be expired
      taosArraySet(pTbUids, vIdx, pUid);
      vIdx++;
    }
  }

  taosArrayPopTailBatch(pTbUids, pTbUids->size - vIdx);

_out:
  TAOS_RETURN(code);
}

static bool ttlMgrNeedFlush(STtlManger *pTtlMgr) {
  return pTtlMgr->flushThreshold > 0 && taosHashGetSize(pTtlMgr->pDirtyUids) > pTtlMgr->flushThreshold;
}

int32_t ttlMgrFlush(STtlManger *pTtlMgr, TXN *pTxn) {
  int64_t startNs = taosGetTimestampNs();
  int64_t endNs = startNs;

  metaTrace("%s, ttl mgr flush start. num of dirty uids:%d", pTtlMgr->logPrefix, taosHashGetSize(pTtlMgr->pDirtyUids));

  int32_t code = TSDB_CODE_SUCCESS;

  void *pIter = NULL;
  while ((pIter = taosHashIterate(pTtlMgr->pDirtyUids, pIter)) != NULL) {
    STtlDirtyEntry *pEntry = (STtlDirtyEntry *)pIter;
    tb_uid_t       *pUid = taosHashGetKey(pIter, NULL);

    STtlCacheEntry *cacheEntry = taosHashGet(pTtlMgr->pTtlCache, pUid, sizeof(*pUid));
    if (cacheEntry == NULL) {
      metaError("%s, ttlMgr flush failed to get ttl cache, uid: %" PRId64 ", type: %d", pTtlMgr->logPrefix, *pUid,
                pEntry->type);
      continue;
    }

    STtlIdxKeyV1 ttlKey;
    ttlMgrBuildKey(&ttlKey, cacheEntry->ttlDays, cacheEntry->changeTimeMs, *pUid);

    STtlIdxKeyV1 ttlKeyDirty;
    ttlMgrBuildKey(&ttlKeyDirty, cacheEntry->ttlDaysDirty, cacheEntry->changeTimeMsDirty, *pUid);

    if (pEntry->type == ENTRY_TYPE_UPSERT) {
      // delete old key & upsert new key
      code = tdbTbDelete(pTtlMgr->pTtlIdx, &ttlKey, sizeof(ttlKey), pTxn);  // maybe first insert, ignore error
      if (TSDB_CODE_SUCCESS != code && TSDB_CODE_NOT_FOUND != code) {
        metaError("%s, ttlMgr flush failed to delete since %s", pTtlMgr->logPrefix, tstrerror(code));
        continue;
      }
      code = tdbTbUpsert(pTtlMgr->pTtlIdx, &ttlKeyDirty, sizeof(ttlKeyDirty), &cacheEntry->ttlDaysDirty,
                         sizeof(cacheEntry->ttlDaysDirty), pTxn);
      if (TSDB_CODE_SUCCESS != code) {
        metaError("%s, ttlMgr flush failed to upsert since %s", pTtlMgr->logPrefix, tstrerror(code));
        continue;
      }

      cacheEntry->ttlDays = cacheEntry->ttlDaysDirty;
      cacheEntry->changeTimeMs = cacheEntry->changeTimeMsDirty;
    } else if (pEntry->type == ENTRY_TYPE_DELETE) {
      code = tdbTbDelete(pTtlMgr->pTtlIdx, &ttlKey, sizeof(ttlKey), pTxn);
      if (TSDB_CODE_SUCCESS != code && TSDB_CODE_NOT_FOUND != code) {
        metaError("%s, ttlMgr flush failed to delete since %s", pTtlMgr->logPrefix, tstrerror(code));
        continue;
      }

      code = taosHashRemove(pTtlMgr->pTtlCache, pUid, sizeof(*pUid));
      if (TSDB_CODE_SUCCESS != code) {
        metaError("%s, ttlMgr flush failed to remove cache since %s", pTtlMgr->logPrefix, tstrerror(code));
        continue;
      }
    } else {
      metaError("%s, ttlMgr flush failed, unknown type: %d", pTtlMgr->logPrefix, pEntry->type);
      continue;
    }

    metaDebug("isdel:%d", pEntry->type == ENTRY_TYPE_DELETE);
    metaDebug("ttlkey:%" PRId64 ", uid:%" PRId64, ttlKey.deleteTimeMs, ttlKey.uid);
    metaDebug("ttlkeyDirty:%" PRId64 ", uid:%" PRId64, ttlKeyDirty.deleteTimeMs, ttlKeyDirty.uid);

    code = taosHashRemove(pTtlMgr->pDirtyUids, pUid, sizeof(*pUid));
    if (TSDB_CODE_SUCCESS != code) {
      metaError("%s, ttlMgr flush failed to remove dirty uid since %s", pTtlMgr->logPrefix, tstrerror(code));
      continue;
    }
  }

  int32_t count = taosHashGetSize(pTtlMgr->pDirtyUids);
  if (count != 0) {
    taosHashClear(pTtlMgr->pDirtyUids);
    metaError("%s, ttlMgr flush failed, dirty uids not empty, count: %d", pTtlMgr->logPrefix, count);
    code = TSDB_CODE_VND_TTL_FLUSH_INCOMPLETION;

    goto _out;
  }

  code = TSDB_CODE_SUCCESS;

_out:
  taosHashCancelIterate(pTtlMgr->pDirtyUids, pIter);

  endNs = taosGetTimestampNs();
  metaTrace("%s, ttl mgr flush end, time consumed: %" PRId64 " ns", pTtlMgr->logPrefix, endNs - startNs);

  TAOS_RETURN(code);
}

1	/*
2	* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3	*
4	* This program is free software: you can use, redistribute, and/or modify
5	* it under the terms of the GNU Affero General Public License, version 3
6	* or later ("AGPL"), as published by the Free Software Foundation.
7	*
8	* This program is distributed in the hope that it will be useful, but WITHOUT
9	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10	* FITNESS FOR A PARTICULAR PURPOSE.
11	*
12	* You should have received a copy of the GNU Affero General Public License
13	* along with this program. If not, see <http://www.gnu.org/licenses/>.
14	*/
15
16	#include "metaTtl.h"
17	#include "meta.h"
18	#include "tdbInt.h"
19
20	typedef struct {
21	TTB *pNewTtlIdx;
22	SMeta *pMeta;
23	} SConvertData;
24
25	typedef struct {
26	int32_t ttlDropMaxCount;
27	int32_t count;
28	STtlIdxKeyV1 expiredKey;
29	SArray *pTbUids;
30	} STtlExpiredCtx;
31
32	static void ttlMgrCleanup(STtlManger *pTtlMgr);
33
34	static int ttlMgrConvert(TTB pOldTtlIdx, TTB pNewTtlIdx, void *pMeta);
35
36	static void ttlMgrBuildKey(STtlIdxKeyV1 *pTtlKey, int64_t ttlDays, int64_t changeTimeMs, tb_uid_t uid);
37	static int ttlIdxKeyCmpr(const void pKey1, int kLen1, const void pKey2, int kLen2);
38	static int ttlIdxKeyV1Cmpr(const void pKey1, int kLen1, const void pKey2, int kLen2);
39	static int ttlMgrFillCache(STtlManger *pTtlMgr);
40	static int32_t ttlMgrFillCacheOneEntry(const void pKey, int keyLen, const void pVal, int valLen, void *pTtlCache);
41	static int32_t ttlMgrConvertOneEntry(const void pKey, int keyLen, const void pVal, int valLen, void *pConvertData);
42	static int32_t ttlMgrFindExpiredOneEntry(const void pKey, int keyLen, const void pVal, int valLen,
43	void *pExpiredInfo);
44
45	static bool ttlMgrNeedFlush(STtlManger *pTtlMgr);
46
47	const char *ttlTbname = "ttl.idx";
48	const char *ttlV1Tbname = "ttlv1.idx";
49
50	int32_t ttlMgrOpen(STtlManger *ppTtlMgr, TDB pEnv, int8_t rollback, const char *logPrefix, int32_t flushThreshold) {	9,556✔
51	int32_t code = TSDB_CODE_SUCCESS;	9,556✔
52	int64_t startNs = taosGetTimestampNs();	9,583✔
53	int32_t pathLen = 0;	9,583✔
54
55	*ppTtlMgr = NULL;	9,583✔
56
57	STtlManger pTtlMgr = (STtlManger )tdbOsCalloc(1, sizeof(*pTtlMgr));	9,583!
58	if (pTtlMgr == NULL) TAOS_RETURN(terrno);	9,581!
59
60	pathLen = strlen(logPrefix) + 1;	9,581✔
61	char logBuffer = (char )tdbOsCalloc(1, pathLen);	9,581!
62	if (logBuffer == NULL) {	9,581!
63	tdbOsFree(pTtlMgr);	×
64	TAOS_RETURN(terrno);	×
65	}
66	tstrncpy(logBuffer, logPrefix, pathLen);	9,581✔
67	pTtlMgr->logPrefix = logBuffer;	9,581✔
68	pTtlMgr->flushThreshold = flushThreshold;	9,581✔
69
70	code = tdbTbOpen(ttlV1Tbname, TDB_VARIANT_LEN, TDB_VARIANT_LEN, ttlIdxKeyV1Cmpr, pEnv, &pTtlMgr->pTtlIdx, rollback);	9,581✔
71	if (TSDB_CODE_SUCCESS != code) {	9,581!
72	metaError("%s, failed to open %s since %s", pTtlMgr->logPrefix, ttlV1Tbname, tstrerror(code));	×
73	tdbOsFree(pTtlMgr);	×
74	TAOS_RETURN(code);	×
75	}
76
77	pTtlMgr->pTtlCache = taosHashInit(8192, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);	9,581✔
78	pTtlMgr->pDirtyUids = taosHashInit(8192, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);	9,583✔
79
80	if ((code = ttlMgrFillCache(pTtlMgr)) != TSDB_CODE_SUCCESS) {	9,583!
81	metaError("%s, failed to fill hash since %s", pTtlMgr->logPrefix, tstrerror(terrno));	×
82	ttlMgrCleanup(pTtlMgr);	×
83	TAOS_RETURN(code);	×
84	}
85
86	int64_t endNs = taosGetTimestampNs();	9,582✔
87	metaInfo("%s, ttl mgr open end, hash size: %d, time consumed: %" PRId64 " ns", pTtlMgr->logPrefix,	9,582!
88	taosHashGetSize(pTtlMgr->pTtlCache), endNs - startNs);
89
90	*ppTtlMgr = pTtlMgr;	9,582✔
91	TAOS_RETURN(TSDB_CODE_SUCCESS);	9,582✔
92	}
93
94	void ttlMgrClose(STtlManger *pTtlMgr) { ttlMgrCleanup(pTtlMgr); }	9,583✔
95
96	bool ttlMgrNeedUpgrade(TDB *pEnv) {	9,559✔
97	bool needUpgrade = tdbTbExist(ttlTbname, pEnv);	9,559✔
98	if (needUpgrade) {	9,559!
99	metaInfo("find ttl idx in old version , will convert");	×
100	}
101	return needUpgrade;	9,559✔
102	}
103
104	int32_t ttlMgrUpgrade(STtlManger pTtlMgr, void pMeta) {	×
105	SMeta meta = (SMeta )pMeta;	×
106	int32_t code = TSDB_CODE_SUCCESS;	×
107
108	if (!tdbTbExist(ttlTbname, meta->pEnv)) TAOS_RETURN(TSDB_CODE_SUCCESS);	×
109
110	metaInfo("%s, ttl mgr start upgrade", pTtlMgr->logPrefix);	×
111
112	int64_t startNs = taosGetTimestampNs();	×
113
114	code = tdbTbOpen(ttlTbname, sizeof(STtlIdxKey), 0, ttlIdxKeyCmpr, meta->pEnv, &pTtlMgr->pOldTtlIdx, 0);	×
115	if (TSDB_CODE_SUCCESS != code) {	×
116	metaError("%s, failed to open %s index since %s", pTtlMgr->logPrefix, ttlTbname, tstrerror(code));	×
117	goto _out;	×
118	}
119
120	if ((code = ttlMgrConvert(pTtlMgr->pOldTtlIdx, pTtlMgr->pTtlIdx, pMeta)) != TSDB_CODE_SUCCESS) {	×
121	metaError("%s, failed to convert ttl index since %s", pTtlMgr->logPrefix, tstrerror(code));	×
122	goto _out;	×
123	}
124
125	if ((code = tdbTbDropByName(ttlTbname, meta->pEnv, meta->txn)) != TSDB_CODE_SUCCESS) {	×
126	metaError("%s, failed to drop old ttl index since %s", pTtlMgr->logPrefix, tstrerror(code));	×
127	goto _out;	×
128	}
129
130	if ((code = ttlMgrFillCache(pTtlMgr)) != TSDB_CODE_SUCCESS) {	×
131	metaError("%s, failed to fill hash since %s", pTtlMgr->logPrefix, tstrerror(code));	×
132	goto _out;	×
133	}
134
135	int64_t endNs = taosGetTimestampNs();	×
136	metaInfo("%s, ttl mgr upgrade end, hash size: %d, time consumed: %" PRId64 " ns", pTtlMgr->logPrefix,	×
137	taosHashGetSize(pTtlMgr->pTtlCache), endNs - startNs);
138
139	_out:	×
140	tdbTbClose(pTtlMgr->pOldTtlIdx);	×
141	pTtlMgr->pOldTtlIdx = NULL;	×
142
143	TAOS_RETURN(code);	×
144	}
145
146	static void ttlMgrCleanup(STtlManger *pTtlMgr) {	9,583✔
147	taosMemoryFree(pTtlMgr->logPrefix);	9,583!
148	taosHashCleanup(pTtlMgr->pTtlCache);	9,583✔
149	taosHashCleanup(pTtlMgr->pDirtyUids);	9,583✔
150	tdbTbClose(pTtlMgr->pTtlIdx);	9,583✔
151	taosMemoryFree(pTtlMgr);	9,583!
152	}	9,583✔
153
154	static void ttlMgrBuildKey(STtlIdxKeyV1 *pTtlKey, int64_t ttlDays, int64_t changeTimeMs, tb_uid_t uid) {	232✔
155	if (ttlDays <= 0) return;	232!
156
157	pTtlKey->deleteTimeMs = changeTimeMs + ttlDays * tsTtlUnit * 1000;	232✔
158	pTtlKey->uid = uid;	232✔
159	}
160
161	static int ttlIdxKeyCmpr(const void pKey1, int kLen1, const void pKey2, int kLen2) {	×
162	STtlIdxKey pTtlIdxKey1 = (STtlIdxKey )pKey1;	×
163	STtlIdxKey pTtlIdxKey2 = (STtlIdxKey )pKey2;	×
164
165	if (pTtlIdxKey1->deleteTimeSec > pTtlIdxKey2->deleteTimeSec) {	×
166	return 1;	×
167	} else if (pTtlIdxKey1->deleteTimeSec < pTtlIdxKey2->deleteTimeSec) {	×
168	return -1;	×
169	}
170
171	if (pTtlIdxKey1->uid > pTtlIdxKey2->uid) {	×
172	return 1;	×
173	} else if (pTtlIdxKey1->uid < pTtlIdxKey2->uid) {	×
174	return -1;	×
175	}
176
177	return 0;	×
178	}
179
180	static int ttlIdxKeyV1Cmpr(const void pKey1, int kLen1, const void pKey2, int kLen2) {	534✔
181	STtlIdxKeyV1 pTtlIdxKey1 = (STtlIdxKeyV1 )pKey1;	534✔
182	STtlIdxKeyV1 pTtlIdxKey2 = (STtlIdxKeyV1 )pKey2;	534✔
183
184	if (pTtlIdxKey1->deleteTimeMs > pTtlIdxKey2->deleteTimeMs) {	534✔
185	return 1;	422✔
186	} else if (pTtlIdxKey1->deleteTimeMs < pTtlIdxKey2->deleteTimeMs) {	112✔
187	return -1;	82✔
188	}
189
190	if (pTtlIdxKey1->uid > pTtlIdxKey2->uid) {	30✔
191	return 1;	9✔
192	} else if (pTtlIdxKey1->uid < pTtlIdxKey2->uid) {	21!
193	return -1;	×
194	}
195
196	return 0;	21✔
197	}
198
199	static int ttlMgrFillCache(STtlManger *pTtlMgr) {	9,583✔
200	return tdbTbTraversal(pTtlMgr->pTtlIdx, pTtlMgr->pTtlCache, ttlMgrFillCacheOneEntry);	9,583✔
201	}
202
203	static int32_t ttlMgrFillCacheOneEntry(const void pKey, int keyLen, const void pVal, int valLen, void *pTtlCache) {	×
204	SHashObj pCache = (SHashObj )pTtlCache;	×
205
206	STtlIdxKeyV1 ttlKey = (STtlIdxKeyV1 )pKey;	×
207	tb_uid_t uid = ttlKey->uid;	×
208	int64_t ttlDays = (int64_t )pVal;	×
209	int64_t changeTimeMs = ttlKey->deleteTimeMs - ttlDays * tsTtlUnit * 1000;	×
210
211	STtlCacheEntry data = {	×
212	.ttlDays = ttlDays, .changeTimeMs = changeTimeMs, .ttlDaysDirty = ttlDays, .changeTimeMsDirty = changeTimeMs};
213
214	return taosHashPut(pCache, &uid, sizeof(uid), &data, sizeof(data));	×
215	}
216
217	static int32_t ttlMgrConvertOneEntry(const void pKey, int keyLen, const void pVal, int valLen, void *pConvertData) {	×
218	SConvertData pData = (SConvertData )pConvertData;	×
219
220	STtlIdxKey ttlKey = (STtlIdxKey )pKey;	×
221	tb_uid_t uid = ttlKey->uid;	×
222	int64_t ttlDays = 0;	×
223
224	int32_t code = TSDB_CODE_SUCCESS;	×
225	if ((code = metaGetTableTtlByUid(pData->pMeta, uid, &ttlDays)) != TSDB_CODE_SUCCESS) {	×
226	metaError("ttlMgr convert failed to get ttl since %s", tstrerror(code));	×
227	goto _out;	×
228	}
229
230	STtlIdxKeyV1 ttlKeyV1 = {.deleteTimeMs = ttlKey->deleteTimeSec * 1000, .uid = uid};	×
231	code = tdbTbUpsert(pData->pNewTtlIdx, &ttlKeyV1, sizeof(ttlKeyV1), &ttlDays, sizeof(ttlDays), pData->pMeta->txn);	×
232	if (code != TSDB_CODE_SUCCESS) {	×
233	metaError("ttlMgr convert failed to upsert since %s", tstrerror(code));	×
234	goto _out;	×
235	}
236
237	code = TSDB_CODE_SUCCESS;	×
238
239	_out:	×
240	TAOS_RETURN(code);	×
241	}
242
243	static int32_t ttlMgrFindExpiredOneEntry(const void pKey, int keyLen, const void pVal, int valLen,	107✔
244	void *pExpiredCtx) {
245	STtlExpiredCtx pCtx = (STtlExpiredCtx )pExpiredCtx;	107✔
246	if (pCtx->count >= pCtx->ttlDropMaxCount) return -1;	107!
247
248	int c = ttlIdxKeyV1Cmpr(&pCtx->expiredKey, sizeof(pCtx->expiredKey), pKey, keyLen);	107✔
249	if (c > 0) {	107✔
250	if (NULL == taosArrayPush(pCtx->pTbUids, &((STtlIdxKeyV1 *)pKey)->uid)) {	164!
251	metaError("ttlMgr find expired failed since %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY));	×
252	return -1;	×
253	}
254	pCtx->count++;	82✔
255	}
256
257	return c;	107✔
258	}
259
260	// static int32_t ttlMgrDumpOneEntry(const void pKey, int keyLen, const void pVal, int valLen, void *pDumpCtx) {
261	// STtlIdxKeyV1 ttlKey = (STtlIdxKeyV1 )pKey;
262	// int64_t ttlDays = (int64_t )pVal;
263
264	// metaInfo("ttlMgr dump, ttl: %" PRId64 ", ctime: %" PRId64 ", uid: %" PRId64, *ttlDays, ttlKey->deleteTimeMs,
265	// ttlKey->uid);
266
267	// TAOS_RETURN(TSDB_CODE_SUCCESS);
268	// }
269
270	static int ttlMgrConvert(TTB pOldTtlIdx, TTB pNewTtlIdx, void *pMeta) {	×
271	SMeta *meta = pMeta;	×
272
273	metaInfo("ttlMgr convert start.");	×
274
275	SConvertData cvData = {.pNewTtlIdx = pNewTtlIdx, .pMeta = meta};	×
276
277	int code = TSDB_CODE_SUCCESS;	×
278	if ((code = tdbTbTraversal(pOldTtlIdx, &cvData, ttlMgrConvertOneEntry)) != TSDB_CODE_SUCCESS) {	×
279	metaError("failed to convert since %s", tstrerror(code));	×
280	}
281
282	metaInfo("ttlMgr convert end.");	×
283	TAOS_RETURN(code);	×
284	}
285
286	int32_t ttlMgrInsertTtl(STtlManger pTtlMgr, const STtlUpdTtlCtx updCtx) {	151,914✔
287	if (updCtx->ttlDays == 0) return 0;	151,914✔
288
289	STtlCacheEntry cacheEntry = {.ttlDays = updCtx->ttlDays,	99✔
290	.changeTimeMs = updCtx->changeTimeMs,	99✔
291	.ttlDaysDirty = updCtx->ttlDays,	99✔
292	.changeTimeMsDirty = updCtx->changeTimeMs};	99✔
293	STtlDirtyEntry dirtryEntry = {.type = ENTRY_TYPE_UPSERT};	99✔
294
295	int32_t code = taosHashPut(pTtlMgr->pTtlCache, &updCtx->uid, sizeof(updCtx->uid), &cacheEntry, sizeof(cacheEntry));	99✔
296	if (TSDB_CODE_SUCCESS != code) {	106!
297	metaError("%s, ttlMgr insert failed to update cache since %s", pTtlMgr->logPrefix, tstrerror(code));	×
298	goto _out;	×
299	}
300
301	code = taosHashPut(pTtlMgr->pDirtyUids, &updCtx->uid, sizeof(updCtx->uid), &dirtryEntry, sizeof(dirtryEntry));	106✔
302	if (TSDB_CODE_SUCCESS != code) {	106!
303	metaError("%s, ttlMgr insert failed to update dirty uids since %s", pTtlMgr->logPrefix, tstrerror(code));	×
304	goto _out;	×
305	}
306
307	if (ttlMgrNeedFlush(pTtlMgr)) {	106!
308	int32_t ret = ttlMgrFlush(pTtlMgr, updCtx->pTxn);	×
309	if (ret < 0) {	×
310	metaError("%s, ttlMgr insert failed to flush since %s", pTtlMgr->logPrefix, tstrerror(ret));	×
311	}
312	}
313
314	code = TSDB_CODE_SUCCESS;	106✔
315
316	_out:	106✔
317	metaTrace("%s, ttl mgr insert ttl, uid: %" PRId64 ", ctime: %" PRId64 ", ttlDays: %" PRId64, pTtlMgr->logPrefix,	106!
318	updCtx->uid, updCtx->changeTimeMs, updCtx->ttlDays);
319
320	TAOS_RETURN(code);	106✔
321	}
322
323	int32_t ttlMgrDeleteTtl(STtlManger pTtlMgr, const STtlDelTtlCtx delCtx) {	17,408✔
324	if (delCtx->ttlDays == 0) return 0;	17,408✔
325
326	STtlDirtyEntry dirtryEntry = {.type = ENTRY_TYPE_DELETE};	52✔
327
328	int32_t code = taosHashPut(pTtlMgr->pDirtyUids, &delCtx->uid, sizeof(delCtx->uid), &dirtryEntry, sizeof(dirtryEntry));	52✔
329	if (TSDB_CODE_SUCCESS != code) {	52!
330	metaError("%s, ttlMgr del failed to update dirty uids since %s", pTtlMgr->logPrefix, tstrerror(code));	×
331	goto _out;	×
332	}
333
334	if (ttlMgrNeedFlush(pTtlMgr)) {	52!
335	int32_t ret = ttlMgrFlush(pTtlMgr, delCtx->pTxn);	×
336	if (ret < 0) {	×
337	metaError("%s, ttlMgr del failed to flush since %s", pTtlMgr->logPrefix, tstrerror(ret));	×
338	}
339	}
340
341	code = TSDB_CODE_SUCCESS;	52✔
342
343	_out:	52✔
344	metaTrace("%s, ttl mgr delete ttl, uid: %" PRId64, pTtlMgr->logPrefix, delCtx->uid);	52!
345	TAOS_RETURN(code);	52✔
346	}
347
348	int32_t ttlMgrUpdateChangeTime(STtlManger pTtlMgr, const STtlUpdCtimeCtx pUpdCtimeCtx) {	3,305✔
349	int32_t code = TSDB_CODE_SUCCESS;	3,305✔
350
351	STtlCacheEntry *oldData = taosHashGet(pTtlMgr->pTtlCache, &pUpdCtimeCtx->uid, sizeof(pUpdCtimeCtx->uid));	3,305✔
352	if (oldData == NULL) {	3,305!
353	goto _out;	3,305✔
354	}
355
356	STtlCacheEntry cacheEntry = {.ttlDays = oldData->ttlDays,	×
357	.changeTimeMs = oldData->changeTimeMs,	×
358	.ttlDaysDirty = oldData->ttlDays,	×
359	.changeTimeMsDirty = pUpdCtimeCtx->changeTimeMs};	×
360	STtlDirtyEntry dirtryEntry = {.type = ENTRY_TYPE_UPSERT};	×
361
362	code =
363	taosHashPut(pTtlMgr->pTtlCache, &pUpdCtimeCtx->uid, sizeof(pUpdCtimeCtx->uid), &cacheEntry, sizeof(cacheEntry));	×
364	if (TSDB_CODE_SUCCESS != code) {	×
365	metaError("%s, ttlMgr update ctime failed to update cache since %s", pTtlMgr->logPrefix, tstrerror(code));	×
366	goto _out;	×
367	}
368
369	code = taosHashPut(pTtlMgr->pDirtyUids, &pUpdCtimeCtx->uid, sizeof(pUpdCtimeCtx->uid), &dirtryEntry,	×
370	sizeof(dirtryEntry));
371	if (TSDB_CODE_SUCCESS != code) {	×
372	metaError("%s, ttlMgr update ctime failed to update dirty uids since %s", pTtlMgr->logPrefix, tstrerror(code));	×
373	goto _out;	×
374	}
375
376	if (ttlMgrNeedFlush(pTtlMgr)) {	×
377	int32_t ret = ttlMgrFlush(pTtlMgr, pUpdCtimeCtx->pTxn);	×
378	if (ret < 0) {	×
379	metaError("%s, ttlMgr update ctime failed to flush since %s", pTtlMgr->logPrefix, tstrerror(ret));	×
380	}
381	}
382
383	code = TSDB_CODE_SUCCESS;	×
384
385	_out:	3,305✔
386	metaTrace("%s, ttl mgr update ctime, uid: %" PRId64 ", ctime: %" PRId64, pTtlMgr->logPrefix, pUpdCtimeCtx->uid,	3,305!
387	pUpdCtimeCtx->changeTimeMs);
388	TAOS_RETURN(code);	3,305✔
389	}
390
391	int32_t ttlMgrFindExpired(STtlManger pTtlMgr, int64_t timePointMs, SArray pTbUids, int32_t ttlDropMaxCount) {	15,681✔
392	int32_t code = TSDB_CODE_SUCCESS;	15,681✔
393
394	STtlIdxKeyV1 ttlKey = {.deleteTimeMs = timePointMs, .uid = INT64_MAX};	15,681✔
395	STtlExpiredCtx expiredCtx = {	15,681✔
396	.ttlDropMaxCount = ttlDropMaxCount, .count = 0, .expiredKey = ttlKey, .pTbUids = pTbUids};
397	TAOS_CHECK_GOTO(tdbTbTraversal(pTtlMgr->pTtlIdx, &expiredCtx, ttlMgrFindExpiredOneEntry), NULL, _out);	15,681!
398
399	size_t vIdx = 0;	15,684✔
400	for (size_t i = 0; i < pTbUids->size; i++) {	15,766✔
401	tb_uid_t *pUid = taosArrayGet(pTbUids, i);	82✔
402	if (taosHashGet(pTtlMgr->pDirtyUids, pUid, sizeof(tb_uid_t)) == NULL) {	82✔
403	// not in dirty && expired in tdb => must be expired
404	taosArraySet(pTbUids, vIdx, pUid);	63✔
405	vIdx++;	63✔
406	}
407	}
408
409	taosArrayPopTailBatch(pTbUids, pTbUids->size - vIdx);	15,684✔
410
411	_out:	15,675✔
412	TAOS_RETURN(code);	15,675✔
413	}
414
415	static bool ttlMgrNeedFlush(STtlManger *pTtlMgr) {	158✔
416	return pTtlMgr->flushThreshold > 0 && taosHashGetSize(pTtlMgr->pDirtyUids) > pTtlMgr->flushThreshold;	158!
417	}
418
419	int32_t ttlMgrFlush(STtlManger pTtlMgr, TXN pTxn) {	13,412✔
420	int64_t startNs = taosGetTimestampNs();	13,413✔
421	int64_t endNs = startNs;	13,413✔
422
423	metaTrace("%s, ttl mgr flush start. num of dirty uids:%d", pTtlMgr->logPrefix, taosHashGetSize(pTtlMgr->pDirtyUids));	13,413✔
424
425	int32_t code = TSDB_CODE_SUCCESS;	13,412✔
426
427	void *pIter = NULL;	13,412✔
428	while ((pIter = taosHashIterate(pTtlMgr->pDirtyUids, pIter)) != NULL) {	13,528✔
429	STtlDirtyEntry pEntry = (STtlDirtyEntry )pIter;	116✔
430	tb_uid_t *pUid = taosHashGetKey(pIter, NULL);	116✔
431
432	STtlCacheEntry cacheEntry = taosHashGet(pTtlMgr->pTtlCache, pUid, sizeof(pUid));	116✔
433	if (cacheEntry == NULL) {	116!
434	metaError("%s, ttlMgr flush failed to get ttl cache, uid: %" PRId64 ", type: %d", pTtlMgr->logPrefix, *pUid,	×
435	pEntry->type);
436	continue;	×
437	}
438
439	STtlIdxKeyV1 ttlKey;
440	ttlMgrBuildKey(&ttlKey, cacheEntry->ttlDays, cacheEntry->changeTimeMs, *pUid);	116✔
441
442	STtlIdxKeyV1 ttlKeyDirty;
443	ttlMgrBuildKey(&ttlKeyDirty, cacheEntry->ttlDaysDirty, cacheEntry->changeTimeMsDirty, *pUid);	116✔
444
445	if (pEntry->type == ENTRY_TYPE_UPSERT) {	116✔
446	// delete old key & upsert new key
447	code = tdbTbDelete(pTtlMgr->pTtlIdx, &ttlKey, sizeof(ttlKey), pTxn); // maybe first insert, ignore error	94✔
448	if (TSDB_CODE_SUCCESS != code && TSDB_CODE_NOT_FOUND != code) {	94!
449	metaError("%s, ttlMgr flush failed to delete since %s", pTtlMgr->logPrefix, tstrerror(code));	×
450	continue;	×
451	}
452	code = tdbTbUpsert(pTtlMgr->pTtlIdx, &ttlKeyDirty, sizeof(ttlKeyDirty), &cacheEntry->ttlDaysDirty,	94✔
453	sizeof(cacheEntry->ttlDaysDirty), pTxn);
454	if (TSDB_CODE_SUCCESS != code) {	94!
455	metaError("%s, ttlMgr flush failed to upsert since %s", pTtlMgr->logPrefix, tstrerror(code));	×
456	continue;	×
457	}
458
459	cacheEntry->ttlDays = cacheEntry->ttlDaysDirty;	94✔
460	cacheEntry->changeTimeMs = cacheEntry->changeTimeMsDirty;	94✔
461	} else if (pEntry->type == ENTRY_TYPE_DELETE) {	22!
462	code = tdbTbDelete(pTtlMgr->pTtlIdx, &ttlKey, sizeof(ttlKey), pTxn);	22✔
463	if (TSDB_CODE_SUCCESS != code && TSDB_CODE_NOT_FOUND != code) {	22!
464	metaError("%s, ttlMgr flush failed to delete since %s", pTtlMgr->logPrefix, tstrerror(code));	×
465	continue;	×
466	}
467
468	code = taosHashRemove(pTtlMgr->pTtlCache, pUid, sizeof(*pUid));	22✔
469	if (TSDB_CODE_SUCCESS != code) {	22!
470	metaError("%s, ttlMgr flush failed to remove cache since %s", pTtlMgr->logPrefix, tstrerror(code));	×
471	continue;	×
472	}
473	} else {
474	metaError("%s, ttlMgr flush failed, unknown type: %d", pTtlMgr->logPrefix, pEntry->type);	×
475	continue;	×
476	}
477
478	metaDebug("isdel:%d", pEntry->type == ENTRY_TYPE_DELETE);	116!
479	metaDebug("ttlkey:%" PRId64 ", uid:%" PRId64, ttlKey.deleteTimeMs, ttlKey.uid);	116!
480	metaDebug("ttlkeyDirty:%" PRId64 ", uid:%" PRId64, ttlKeyDirty.deleteTimeMs, ttlKeyDirty.uid);	116!
481
482	code = taosHashRemove(pTtlMgr->pDirtyUids, pUid, sizeof(*pUid));	116✔
483	if (TSDB_CODE_SUCCESS != code) {	116!
484	metaError("%s, ttlMgr flush failed to remove dirty uid since %s", pTtlMgr->logPrefix, tstrerror(code));	×
485	continue;	×
486	}
487	}
488
489	int32_t count = taosHashGetSize(pTtlMgr->pDirtyUids);	13,413✔
490	if (count != 0) {	13,413!
491	taosHashClear(pTtlMgr->pDirtyUids);	×
492	metaError("%s, ttlMgr flush failed, dirty uids not empty, count: %d", pTtlMgr->logPrefix, count);	×
493	code = TSDB_CODE_VND_TTL_FLUSH_INCOMPLETION;	×
494
495	goto _out;	×
496	}
497
498	code = TSDB_CODE_SUCCESS;	13,413✔
499
500	_out:	13,413✔
501	taosHashCancelIterate(pTtlMgr->pDirtyUids, pIter);	13,413✔
502
503	endNs = taosGetTimestampNs();	13,413✔
504	metaTrace("%s, ttl mgr flush end, time consumed: %" PRId64 " ns", pTtlMgr->logPrefix, endNs - startNs);	13,413✔
505
506	TAOS_RETURN(code);	13,413✔
507	}

taosdata / TDengine / #4808

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous