• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

taosdata / TDengine / #3558

17 Dec 2024 06:05AM UTC coverage: 59.778% (+1.6%) from 58.204%
#3558

push

travis-ci

web-flow
Merge pull request #29179 from taosdata/merge/mainto3.0

merge: form main to 3.0 branch

132787 of 287595 branches covered (46.17%)

Branch coverage included in aggregate %.

104 of 191 new or added lines in 5 files covered. (54.45%)

6085 existing lines in 168 files now uncovered.

209348 of 284746 relevant lines covered (73.52%)

8164844.48 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

64.61
/source/util/src/tscalablebf.c
1
/*
2
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3
 *
4
 * This program is free software: you can use, redistribute, and/or modify
5
 * it under the terms of the GNU Affero General Public License, version 3
6
 * or later ("AGPL"), as published by the Free Software Foundation.
7
 *
8
 * This program is distributed in the hope that it will be useful, but WITHOUT
9
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10
 * FITNESS FOR A PARTICULAR PURPOSE.
11
 *
12
 * You should have received a copy of the GNU Affero General Public License
13
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
14
 */
15

16
#define _DEFAULT_SOURCE
17

18
#include "tscalablebf.h"
19
#include "taoserror.h"
20
#include "tutil.h"
21

22
#define DEFAULT_GROWTH           2
23
#define DEFAULT_TIGHTENING_RATIO 0.5
24
#define DEFAULT_MAX_BLOOMFILTERS 4
25
#define SBF_INVALID              -1
26
#define SBF_VALID                0
27

28
static int32_t tScalableBfAddFilter(SScalableBf* pSBf, uint64_t expectedEntries, double errorRate,
29
                                    SBloomFilter** ppNormalBf);
30

31
int32_t tScalableBfInit(uint64_t expectedEntries, double errorRate, SScalableBf** ppSBf) {
94,700,798✔
32
  int32_t        code = TSDB_CODE_SUCCESS;
94,700,798✔
33
  int32_t        lino = 0;
94,700,798✔
34
  const uint32_t defaultSize = 8;
94,700,798✔
35
  if (expectedEntries < 1 || errorRate <= 0 || errorRate >= 1.0) {
94,700,798!
36
    code = TSDB_CODE_INVALID_PARA;
×
37
    QUERY_CHECK_CODE(code, lino, _error);
×
38
  }
39
  SScalableBf* pSBf = taosMemoryCalloc(1, sizeof(SScalableBf));
94,700,794!
40
  if (pSBf == NULL) {
98,143,245!
41
    code = terrno;
×
42
    QUERY_CHECK_CODE(code, lino, _error);
×
43
  }
44
  pSBf->maxBloomFilters = DEFAULT_MAX_BLOOMFILTERS;
98,143,245✔
45
  pSBf->status = SBF_VALID;
98,143,245✔
46
  pSBf->numBits = 0;
98,143,245✔
47
  pSBf->bfArray = taosArrayInit(defaultSize, sizeof(void*));
98,143,245✔
48
  if (!pSBf->bfArray) {
99,525,113!
49
    code = terrno;
×
50
    QUERY_CHECK_CODE(code, lino, _error);
×
51
  }
52

53
  SBloomFilter* pNormalBf = NULL;
99,525,113✔
54
  code = tScalableBfAddFilter(pSBf, expectedEntries, errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf);
99,525,113✔
55
  if (code != TSDB_CODE_SUCCESS) {
98,842,204!
56
    tScalableBfDestroy(pSBf);
×
57
    QUERY_CHECK_CODE(code, lino, _error);
×
58
  }
59
  pSBf->growth = DEFAULT_GROWTH;
98,849,286✔
60
  pSBf->hashFn1 = HASH_FUNCTION_1;
98,849,286✔
61
  pSBf->hashFn2 = HASH_FUNCTION_2;
98,849,286✔
62
  (*ppSBf) = pSBf;
98,849,286✔
63
  return TSDB_CODE_SUCCESS;
98,849,286✔
64

65
_error:
4✔
66
  uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
4!
67
  return code;
4✔
68
}
69

70
int32_t tScalableBfPutNoCheck(SScalableBf* pSBf, const void* keyBuf, uint32_t len) {
14,593,617✔
71
  int32_t code = TSDB_CODE_SUCCESS;
14,593,617✔
72
  int32_t lino = 0;
14,593,617✔
73
  if (pSBf->status == SBF_INVALID) {
14,593,617✔
74
    return code;
12,987,253✔
75
  }
76
  int32_t       size = taosArrayGetSize(pSBf->bfArray);
1,606,364✔
77
  SBloomFilter* pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1);
1,606,596✔
78
  if (!pNormalBf) {
1,606,398!
79
    code = terrno;
×
80
    QUERY_CHECK_CODE(code, lino, _error);
×
81
  }
82
  if (tBloomFilterIsFull(pNormalBf)) {
1,606,398✔
83
    code = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth,
39✔
84
                                pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf);
39✔
85
    if (code != TSDB_CODE_SUCCESS) {
39✔
86
      pSBf->status = SBF_INVALID;
7✔
87
      if (code == TSDB_CODE_OUT_OF_BUFFER) {
7!
88
        code = TSDB_CODE_SUCCESS;
7✔
89
        return code;
7✔
90
      }
91
      QUERY_CHECK_CODE(code, lino, _error);
×
92
    }
93
  }
94
  return tBloomFilterPut(pNormalBf, keyBuf, len);
1,606,545✔
95

96
_error:
×
97
  if (code != TSDB_CODE_SUCCESS) {
×
98
    uDebug("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
99
  }
100
  return code;
×
101
}
102

103
int32_t tScalableBfPut(SScalableBf* pSBf, const void* keyBuf, uint32_t len, int32_t* winRes) {
61,131,026✔
104
  int32_t code = TSDB_CODE_SUCCESS;
61,131,026✔
105
  int32_t lino = 0;
61,131,026✔
106
  if (pSBf->status == SBF_INVALID) {
61,131,026✔
107
    (*winRes) = TSDB_CODE_FAILED;
51,840,959✔
108
    return code;
51,840,959✔
109
  }
110
  uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len);
9,290,067✔
111
  uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len);
9,356,846✔
112
  int32_t  size = taosArrayGetSize(pSBf->bfArray);
9,384,876✔
113
  for (int32_t i = size - 2; i >= 0; --i) {
15,927,350✔
114
    if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i), h1, h2) != TSDB_CODE_SUCCESS) {
6,741,962✔
115
      (*winRes) = TSDB_CODE_FAILED;
174,992✔
116
      goto _end;
174,992✔
117
    }
118
  }
119

120
  SBloomFilter* pNormalBf = taosArrayGetP(pSBf->bfArray, size - 1);
9,185,388✔
121
  QUERY_CHECK_NULL(pNormalBf, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
9,162,322!
122

123
  if (tBloomFilterIsFull(pNormalBf)) {
9,162,322✔
124
    code = tScalableBfAddFilter(pSBf, pNormalBf->expectedEntries * pSBf->growth,
79✔
125
                                pNormalBf->errorRate * DEFAULT_TIGHTENING_RATIO, &pNormalBf);
79✔
126
    if (code != TSDB_CODE_SUCCESS) {
79✔
127
      pSBf->status = SBF_INVALID;
19✔
128
      if (code == TSDB_CODE_OUT_OF_BUFFER) {
19!
129
        code = TSDB_CODE_SUCCESS;
19✔
130
        (*winRes) = TSDB_CODE_FAILED;
19✔
131
        goto _end;
19✔
132
      }
133
      QUERY_CHECK_CODE(code, lino, _end);
×
134
    }
135
  }
136
  (*winRes) = tBloomFilterPutHash(pNormalBf, h1, h2);
9,160,610✔
137

138
_end:
9,325,288✔
139
  if (code != TSDB_CODE_SUCCESS) {
9,325,288!
140
    uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
141
  }
142
  return code;
9,319,224✔
143
}
144

145
int32_t tScalableBfNoContain(const SScalableBf* pSBf, const void* keyBuf, uint32_t len) {
3,589✔
146
  if (pSBf->status == SBF_INVALID) {
3,589!
147
    return TSDB_CODE_FAILED;
×
148
  }
149
  uint64_t h1 = (uint64_t)pSBf->hashFn1(keyBuf, len);
3,589✔
150
  uint64_t h2 = (uint64_t)pSBf->hashFn2(keyBuf, len);
3,589✔
151
  int32_t  size = taosArrayGetSize(pSBf->bfArray);
3,589✔
152
  for (int32_t i = size - 1; i >= 0; --i) {
5,816✔
153
    if (tBloomFilterNoContain(taosArrayGetP(pSBf->bfArray, i), h1, h2) != TSDB_CODE_SUCCESS) {
4,816✔
154
      return TSDB_CODE_FAILED;
2,589✔
155
    }
156
  }
157
  return TSDB_CODE_SUCCESS;
1,000✔
158
}
159

160
static int32_t tScalableBfAddFilter(SScalableBf* pSBf, uint64_t expectedEntries, double errorRate,
99,413,042✔
161
                                    SBloomFilter** ppNormalBf) {
162
  int32_t code = TSDB_CODE_SUCCESS;
99,413,042✔
163
  int32_t lino = 0;
99,413,042✔
164
  if (taosArrayGetSize(pSBf->bfArray) >= pSBf->maxBloomFilters) {
99,413,042✔
165
    code = TSDB_CODE_OUT_OF_BUFFER;
26✔
166
    QUERY_CHECK_CODE(code, lino, _error);
26!
167
  }
168

169
  SBloomFilter* pNormalBf = NULL;
99,038,445✔
170
  code = tBloomFilterInit(expectedEntries, errorRate, &pNormalBf);
99,038,445✔
171
  QUERY_CHECK_CODE(code, lino, _error);
100,799,405!
172

173
  if (taosArrayPush(pSBf->bfArray, &pNormalBf) == NULL) {
199,815,851!
174
    tBloomFilterDestroy(pNormalBf);
×
175
    code = terrno;
×
UNCOV
176
    QUERY_CHECK_CODE(code, lino, _error);
×
177
  }
178
  pSBf->numBits += pNormalBf->numBits;
98,883,710✔
179
  (*ppNormalBf) = pNormalBf;
98,883,710✔
180

181
_error:
98,883,736✔
182
  if (code != TSDB_CODE_SUCCESS) {
98,883,736✔
183
    uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
26!
184
  }
185
  return code;
98,891,453✔
186
}
187

188
void tScalableBfDestroy(SScalableBf* pSBf) {
102,467,542✔
189
  if (pSBf == NULL) {
102,467,542!
190
    return;
×
191
  }
192
  if (pSBf->bfArray != NULL) {
102,467,542!
193
    taosArrayDestroyP(pSBf->bfArray, (FDelete)tBloomFilterDestroy);
102,477,427✔
194
  }
195
  taosMemoryFree(pSBf);
102,773,392!
196
}
197

198
int32_t tScalableBfEncode(const SScalableBf* pSBf, SEncoder* pEncoder) {
49,116✔
199
  if (!pSBf) {
49,116✔
200
    TAOS_CHECK_RETURN(tEncodeI32(pEncoder, 0));
3,428!
201
    return 0;
3,428✔
202
  }
203
  int32_t size = taosArrayGetSize(pSBf->bfArray);
45,688✔
204
  TAOS_CHECK_RETURN(tEncodeI32(pEncoder, size));
45,686!
205
  for (int32_t i = 0; i < size; i++) {
91,443✔
206
    SBloomFilter* pBF = taosArrayGetP(pSBf->bfArray, i);
45,741✔
207
    TAOS_CHECK_RETURN(tBloomFilterEncode(pBF, pEncoder));
45,712!
208
  }
209
  TAOS_CHECK_RETURN(tEncodeU32(pEncoder, pSBf->growth));
91,404!
210
  TAOS_CHECK_RETURN(tEncodeU64(pEncoder, pSBf->numBits));
91,404!
211
  TAOS_CHECK_RETURN(tEncodeU32(pEncoder, pSBf->maxBloomFilters));
91,404!
212
  TAOS_CHECK_RETURN(tEncodeI8(pEncoder, pSBf->status));
91,404!
213
  return 0;
45,702✔
214
}
215

216
int32_t tScalableBfDecode(SDecoder* pDecoder, SScalableBf** ppSBf) {
24,522✔
217
  int32_t      code = TSDB_CODE_SUCCESS;
24,522✔
218
  int32_t      lino = 0;
24,522✔
219
  SScalableBf* pSBf = taosMemoryCalloc(1, sizeof(SScalableBf));
24,522!
220
  if (!pSBf) {
24,527!
221
    code = terrno;
×
222
    QUERY_CHECK_CODE(code, lino, _error);
×
223
  }
224
  pSBf->hashFn1 = HASH_FUNCTION_1;
24,527✔
225
  pSBf->hashFn2 = HASH_FUNCTION_2;
24,527✔
226
  pSBf->bfArray = NULL;
24,527✔
227
  int32_t size = 0;
24,527✔
228
  if (tDecodeI32(pDecoder, &size) < 0) {
24,527!
229
    code = terrno;
×
230
    QUERY_CHECK_CODE(code, lino, _error);
×
231
  }
232
  if (size == 0) {
24,527✔
233
    (*ppSBf) = NULL;
1,694✔
234
    tScalableBfDestroy(pSBf);
1,694✔
235
    goto _error;
1,694✔
236
  }
237
  pSBf->bfArray = taosArrayInit(size * 2, POINTER_BYTES);
22,833✔
238
  if (!pSBf->bfArray) {
22,833!
239
    code = terrno;
×
240
    QUERY_CHECK_CODE(code, lino, _error);
×
241
  }
242

243
  for (int32_t i = 0; i < size; i++) {
45,680✔
244
    SBloomFilter* pBF = NULL;
22,848✔
245
    code = tBloomFilterDecode(pDecoder, &pBF);
22,848✔
246
    QUERY_CHECK_CODE(code, lino, _error);
22,847!
247
    void* tmpRes = taosArrayPush(pSBf->bfArray, &pBF);
22,847✔
248
    if (!tmpRes) {
22,847!
249
      code = terrno;
×
250
      QUERY_CHECK_CODE(code, lino, _error);
×
251
    }
252
  }
253
  if (tDecodeU32(pDecoder, &pSBf->growth) < 0) {
45,664!
254
    code = terrno;
×
255
    QUERY_CHECK_CODE(code, lino, _error);
×
256
  }
257
  if (tDecodeU64(pDecoder, &pSBf->numBits) < 0) {
45,662!
258
    code = terrno;
×
259
    QUERY_CHECK_CODE(code, lino, _error);
×
260
  }
261
  if (tDecodeU32(pDecoder, &pSBf->maxBloomFilters) < 0) {
68,489!
262
    code = terrno;
×
263
    QUERY_CHECK_CODE(code, lino, _error);
×
264
  }
265
  if (tDecodeI8(pDecoder, &pSBf->status) < 0) {
45,658!
266
    code = terrno;
×
267
    QUERY_CHECK_CODE(code, lino, _error);
×
268
  }
269
  (*ppSBf) = pSBf;
22,829✔
270

271
_error:
24,523✔
272

273
  if (code != TSDB_CODE_SUCCESS) {
24,523!
274
    tScalableBfDestroy(pSBf);
×
275
    uError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
276
  }
277
  return code;
24,523✔
278
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc