#3599

Committed 08 Feb 2025 11:23AM UTC coverage: 1.77% (-61.6%) from 63.396%

Build # #3599

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

Merge pull request #29712 from taosdata/fix/TD-33652-3.0

fix: reduce write rows from 30w to 3w

Run Details

3776 of 278949 branches covered (1.35%)

Branch coverage included in aggregate %.

6012 of 274147 relevant lines covered (2.19%)

1642.73 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c

/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#include "tsdbFSetRAW.h"
#include "tsdbFS2.h"

// SFSetRAWWriter ==================================================
typedef struct SFSetRAWWriter {
  SFSetRAWWriterConfig config[1];

  struct {
    TFileOpArray fopArr[1];
    STFile       file;
    int64_t      offset;
  } ctx[1];

  // writer
  SDataFileRAWWriter *dataWriter;
} SFSetRAWWriter;

int32_t tsdbFSetRAWWriterOpen(SFSetRAWWriterConfig *config, SFSetRAWWriter **writer) {
  int32_t code = 0;
  int32_t lino = 0;

  writer[0] = taosMemoryCalloc(1, sizeof(SFSetRAWWriter));
  if (writer[0] == NULL) {
    return terrno;
  }

  writer[0]->config[0] = config[0];

  TARRAY2_INIT(writer[0]->ctx->fopArr);

_exit:
  if (code) {
    TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code);
  }
  return code;
}

static int32_t tsdbFSetRAWWriterFinish(SFSetRAWWriter *writer, TFileOpArray *fopArr) {
  int32_t code = 0;
  int32_t lino = 0;

  STsdb *tsdb = writer->config->tsdb;

  STFileOp op;
  TARRAY2_FOREACH(writer->ctx->fopArr, op) {
    code = TARRAY2_APPEND(fopArr, op);
    TSDB_CHECK_CODE(code, lino, _exit);
  }

  TARRAY2_CLEAR(writer->ctx->fopArr, NULL);
_exit:
  if (code) {
    TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);
  }
  return code;
}

static int32_t tsdbFSetRAWWriteFileDataBegin(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr) {
  int32_t code = 0;
  int32_t lino = 0;

  SDataFileRAWWriterConfig config = {
      .tsdb = writer->config->tsdb,
      .szPage = writer->config->szPage,
      .fid = bHdr->file.fid,
      .did = writer->config->did,
      .cid = bHdr->file.cid,
      .level = writer->config->level,

      .file =
          {
              .type = bHdr->file.type,
              .fid = bHdr->file.fid,
              .did = writer->config->did,
              .cid = bHdr->file.cid,
              .size = bHdr->file.size,
              .minVer = bHdr->file.minVer,
              .maxVer = bHdr->file.maxVer,
              .stt = {{
                  .level = bHdr->file.stt->level,
              }},
          },
  };

  tsdbFSUpdateEid(config.tsdb->pFS, config.cid);
  writer->ctx->offset = 0;
  writer->ctx->file = config.file;

  code = tsdbDataFileRAWWriterOpen(&config, &writer->dataWriter);
  TSDB_CHECK_CODE(code, lino, _exit);

_exit:
  if (code) {
    TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
  }
  return code;
}

static int32_t tsdbFSetRAWWriteFileDataEnd(SFSetRAWWriter *writer) {
  int32_t code = 0;
  int32_t lino = 0;

  code = tsdbDataFileRAWWriterClose(&writer->dataWriter, false, writer->ctx->fopArr);
  TSDB_CHECK_CODE(code, lino, _exit);

_exit:
  if (code) {
    TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
  }
  return code;
}

int32_t tsdbFSetRAWWriterClose(SFSetRAWWriter **writer, bool abort, TFileOpArray *fopArr) {
  if (writer[0] == NULL) return 0;

  int32_t code = 0;
  int32_t lino = 0;

  STsdb *tsdb = writer[0]->config->tsdb;

  // end
  code = tsdbFSetRAWWriteFileDataEnd(writer[0]);
  TSDB_CHECK_CODE(code, lino, _exit);

  code = tsdbDataFileRAWWriterClose(&writer[0]->dataWriter, abort, writer[0]->ctx->fopArr);
  TSDB_CHECK_CODE(code, lino, _exit);

  code = tsdbFSetRAWWriterFinish(writer[0], fopArr);
  TSDB_CHECK_CODE(code, lino, _exit);
  // free
  TARRAY2_DESTROY(writer[0]->ctx->fopArr, NULL);
  taosMemoryFree(writer[0]);
  writer[0] = NULL;

_exit:
  if (code) {
    TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);
  }
  return code;
}

int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter *writer, STsdbDataRAWBlockHeader *bHdr, int32_t encryptAlgorithm,
                                  char *encryptKey) {
  int32_t code = 0;
  int32_t lino = 0;

  if (writer->ctx->offset == writer->ctx->file.size) {
    code = tsdbFSetRAWWriteFileDataEnd(writer);
    TSDB_CHECK_CODE(code, lino, _exit);

    code = tsdbFSetRAWWriteFileDataBegin(writer, bHdr);
    TSDB_CHECK_CODE(code, lino, _exit);
  }

  code = tsdbDataFileRAWWriteBlockData(writer->dataWriter, bHdr, encryptAlgorithm, encryptKey);
  TSDB_CHECK_CODE(code, lino, _exit);

  writer->ctx->offset += bHdr->dataLength;

_exit:
  if (code) {
    TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);
  }
  return code;
}

1	/*
2	* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3	*
4	* This program is free software: you can use, redistribute, and/or modify
5	* it under the terms of the GNU Affero General Public License, version 3
6	* or later ("AGPL"), as published by the Free Software Foundation.
7	*
8	* This program is distributed in the hope that it will be useful, but WITHOUT
9	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10	* FITNESS FOR A PARTICULAR PURPOSE.
11	*
12	* You should have received a copy of the GNU Affero General Public License
13	* along with this program. If not, see <http://www.gnu.org/licenses/>.
14	*/
15
16	#include "tsdbFSetRAW.h"
17	#include "tsdbFS2.h"
18
19	// SFSetRAWWriter ==================================================
20	typedef struct SFSetRAWWriter {
21	SFSetRAWWriterConfig config[1];
22
23	struct {
24	TFileOpArray fopArr[1];
25	STFile file;
26	int64_t offset;
27	} ctx[1];
28
29	// writer
30	SDataFileRAWWriter *dataWriter;
31	} SFSetRAWWriter;
32
33	int32_t tsdbFSetRAWWriterOpen(SFSetRAWWriterConfig config, SFSetRAWWriter *writer) {	×
34	int32_t code = 0;	×
35	int32_t lino = 0;	×
36
37	writer[0] = taosMemoryCalloc(1, sizeof(SFSetRAWWriter));	×
38	if (writer[0] == NULL) {	×
39	return terrno;	×
40	}
41
42	writer[0]->config[0] = config[0];	×
43
44	TARRAY2_INIT(writer[0]->ctx->fopArr);	×
45
46	_exit:	×
47	if (code) {	×
48	TSDB_ERROR_LOG(TD_VID(config->tsdb->pVnode), lino, code);	×
49	}
50	return code;	×
51	}
52
53	static int32_t tsdbFSetRAWWriterFinish(SFSetRAWWriter writer, TFileOpArray fopArr) {	×
54	int32_t code = 0;	×
55	int32_t lino = 0;	×
56
57	STsdb *tsdb = writer->config->tsdb;	×
58
59	STFileOp op;
60	TARRAY2_FOREACH(writer->ctx->fopArr, op) {	×
61	code = TARRAY2_APPEND(fopArr, op);	×
62	TSDB_CHECK_CODE(code, lino, _exit);	×
63	}
64
65	TARRAY2_CLEAR(writer->ctx->fopArr, NULL);	×
66	_exit:	×
67	if (code) {	×
68	TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);	×
69	}
70	return code;	×
71	}
72
73	static int32_t tsdbFSetRAWWriteFileDataBegin(SFSetRAWWriter writer, STsdbDataRAWBlockHeader bHdr) {	×
74	int32_t code = 0;	×
75	int32_t lino = 0;	×
76
77	SDataFileRAWWriterConfig config = {	×
78	.tsdb = writer->config->tsdb,	×
79	.szPage = writer->config->szPage,	×
80	.fid = bHdr->file.fid,	×
81	.did = writer->config->did,
82	.cid = bHdr->file.cid,	×
83	.level = writer->config->level,	×
84
85	.file =
86	{
87	.type = bHdr->file.type,	×
88	.fid = bHdr->file.fid,	×
89	.did = writer->config->did,
90	.cid = bHdr->file.cid,	×
91	.size = bHdr->file.size,	×
92	.minVer = bHdr->file.minVer,	×
93	.maxVer = bHdr->file.maxVer,	×
94	.stt = {{
95	.level = bHdr->file.stt->level,	×
96	}},
97	},
98	};
99
100	tsdbFSUpdateEid(config.tsdb->pFS, config.cid);	×
101	writer->ctx->offset = 0;	×
102	writer->ctx->file = config.file;	×
103
104	code = tsdbDataFileRAWWriterOpen(&config, &writer->dataWriter);	×
105	TSDB_CHECK_CODE(code, lino, _exit);	×
106
107	_exit:	×
108	if (code) {	×
109	TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);	×
110	}
111	return code;	×
112	}
113
114	static int32_t tsdbFSetRAWWriteFileDataEnd(SFSetRAWWriter *writer) {	×
115	int32_t code = 0;	×
116	int32_t lino = 0;	×
117
118	code = tsdbDataFileRAWWriterClose(&writer->dataWriter, false, writer->ctx->fopArr);	×
119	TSDB_CHECK_CODE(code, lino, _exit);	×
120
121	_exit:	×
122	if (code) {	×
123	TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);	×
124	}
125	return code;	×
126	}
127
128	int32_t tsdbFSetRAWWriterClose(SFSetRAWWriter *writer, bool abort, TFileOpArray fopArr) {	×
129	if (writer[0] == NULL) return 0;	×
130
131	int32_t code = 0;	×
132	int32_t lino = 0;	×
133
134	STsdb *tsdb = writer[0]->config->tsdb;	×
135
136	// end
137	code = tsdbFSetRAWWriteFileDataEnd(writer[0]);	×
138	TSDB_CHECK_CODE(code, lino, _exit);	×
139
140	code = tsdbDataFileRAWWriterClose(&writer[0]->dataWriter, abort, writer[0]->ctx->fopArr);	×
141	TSDB_CHECK_CODE(code, lino, _exit);	×
142
143	code = tsdbFSetRAWWriterFinish(writer[0], fopArr);	×
144	TSDB_CHECK_CODE(code, lino, _exit);	×
145	// free
146	TARRAY2_DESTROY(writer[0]->ctx->fopArr, NULL);	×
147	taosMemoryFree(writer[0]);	×
148	writer[0] = NULL;	×
149
150	_exit:	×
151	if (code) {	×
152	TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code);	×
153	}
154	return code;	×
155	}
156
157	int32_t tsdbFSetRAWWriteBlockData(SFSetRAWWriter writer, STsdbDataRAWBlockHeader bHdr, int32_t encryptAlgorithm,	×
158	char *encryptKey) {
159	int32_t code = 0;	×
160	int32_t lino = 0;	×
161
162	if (writer->ctx->offset == writer->ctx->file.size) {	×
163	code = tsdbFSetRAWWriteFileDataEnd(writer);	×
164	TSDB_CHECK_CODE(code, lino, _exit);	×
165
166	code = tsdbFSetRAWWriteFileDataBegin(writer, bHdr);	×
167	TSDB_CHECK_CODE(code, lino, _exit);	×
168	}
169
170	code = tsdbDataFileRAWWriteBlockData(writer->dataWriter, bHdr, encryptAlgorithm, encryptKey);	×
171	TSDB_CHECK_CODE(code, lino, _exit);	×
172
173	writer->ctx->offset += bHdr->dataLength;	×
174
175	_exit:	×
176	if (code) {	×
177	TSDB_ERROR_LOG(TD_VID(writer->config->tsdb->pVnode), lino, code);	×
178	}
179	return code;	×
180	}

taosdata / TDengine / #3599

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous