14630481637

Committed 23 Apr 2025 07:04PM UTC coverage: 72.178% (-0.002%) from 72.18%

Build # 14630481637

Build Type

push

github

Committed by

DaanDeMeyer

Commit Message

mkosi: Run clangd within the tools tree instead of the build container

Running within the build sandbox has a number of disadvantages:
- We have a separate clangd cache for each distribution/release combo
- It requires to build the full image before clangd can be used
- It breaks every time the image becomes out of date and requires a
  rebuild
- We can't look at system headers as we don't have the knowledge to map
  them from inside the build sandbox to the corresponding path on the host

Instead, let's have mkosi.clangd run clangd within the tools tree. We
already require building systemd for both the host and the target anyway,
and all the dependencies to build systemd are installed in the tools tree
already for that, as well as clangd since it's installed together with the
other clang tooling we install in the tools tree. Unlike the previous approach,
this approach only requires the mkosi tools tree to be built upfront, which has
a much higher chance of not invalidating its cache. We can also trivially map
system header lookups from within the sandbox to the path within mkosi.tools
on the host so that starts working as well.

Coverage Stats

297054 of 411557 relevant lines covered (72.18%)

686269.58 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

72.08

/src/import/export-tar.c

/* SPDX-License-Identifier: LGPL-2.1-or-later */

#include "sd-daemon.h"

#include "alloc-util.h"
#include "btrfs-util.h"
#include "export-tar.h"
#include "fd-util.h"
#include "import-common.h"
#include "log.h"
#include "pretty-print.h"
#include "process-util.h"
#include "ratelimit.h"
#include "string-util.h"
#include "tmpfile-util.h"

#define COPY_BUFFER_SIZE (16*1024)

struct TarExport {
        sd_event *event;

        TarExportFinished on_finished;
        void *userdata;

        char *path;
        char *temp_path;

        int output_fd;
        int tar_fd;

        ImportCompress compress;

        sd_event_source *output_event_source;

        void *buffer;
        size_t buffer_size;
        size_t buffer_allocated;

        uint64_t written_compressed;
        uint64_t written_uncompressed;

        pid_t tar_pid;

        struct stat st;
        uint64_t quota_referenced;

        unsigned last_percent;
        RateLimit progress_ratelimit;

        bool eof;
        bool tried_splice;
};

TarExport *tar_export_unref(TarExport *e) {
        if (!e)
                return NULL;

        sd_event_source_unref(e->output_event_source);

        if (e->tar_pid > 1)
                sigkill_wait(e->tar_pid);

        if (e->temp_path) {
                (void) btrfs_subvol_remove(e->temp_path, BTRFS_REMOVE_QUOTA);
                free(e->temp_path);
        }

        import_compress_free(&e->compress);

        sd_event_unref(e->event);

        safe_close(e->tar_fd);

        free(e->buffer);
        free(e->path);
        return mfree(e);
}

int tar_export_new(
                TarExport **ret,
                sd_event *event,
                TarExportFinished on_finished,
                void *userdata) {

        _cleanup_(tar_export_unrefp) TarExport *e = NULL;
        int r;

        assert(ret);

        e = new(TarExport, 1);
        if (!e)
                return -ENOMEM;

        *e = (TarExport) {
                .output_fd = -EBADF,
                .tar_fd = -EBADF,
                .on_finished = on_finished,
                .userdata = userdata,
                .quota_referenced = UINT64_MAX,
                .last_percent = UINT_MAX,
                .progress_ratelimit = { 100 * USEC_PER_MSEC, 1 },
        };

        if (event)
                e->event = sd_event_ref(event);
        else {
                r = sd_event_default(&e->event);
                if (r < 0)
                        return r;
        }

        *ret = TAKE_PTR(e);

        return 0;
}

static void tar_export_report_progress(TarExport *e) {
        unsigned percent;
        assert(e);

        /* Do we have any quota info? If not, we don't know anything about the progress */
        if (e->quota_referenced == UINT64_MAX)
                return;

        if (e->written_uncompressed >= e->quota_referenced)
                percent = 100;
        else
                percent = (unsigned) ((e->written_uncompressed * UINT64_C(100)) / e->quota_referenced);

        if (percent == e->last_percent)
                return;

        if (!ratelimit_below(&e->progress_ratelimit))
                return;

        sd_notifyf(false, "X_IMPORT_PROGRESS=%u%%", percent);

        if (isatty_safe(STDERR_FILENO))
                (void) draw_progress_barf(
                                percent,
                                "%s %s/%s",
                                glyph(GLYPH_ARROW_RIGHT),
                                FORMAT_BYTES(e->written_uncompressed),
                                FORMAT_BYTES(e->quota_referenced));
        else
                log_info("Exported %u%%.", percent);

        e->last_percent = percent;
}

static int tar_export_finish(TarExport *e) {
        int r;

        assert(e);
        assert(e->tar_fd >= 0);

        if (e->tar_pid > 0) {
                r = wait_for_terminate_and_check("tar", TAKE_PID(e->tar_pid), WAIT_LOG);
                if (r < 0)
                        return r;
                if (r != EXIT_SUCCESS)
                        return -EPROTO;
        }

        e->tar_fd = safe_close(e->tar_fd);

        return 0;
}

static int tar_export_process(TarExport *e) {
        ssize_t l;
        int r;

        assert(e);

        if (!e->tried_splice && e->compress.type == IMPORT_COMPRESS_UNCOMPRESSED) {

                l = splice(e->tar_fd, NULL, e->output_fd, NULL, COPY_BUFFER_SIZE, 0);
                if (l < 0) {
                        if (errno == EAGAIN)
                                return 0;

                        e->tried_splice = true;
                } else if (l == 0) {
                        r = tar_export_finish(e);
                        goto finish;
                } else {
                        e->written_uncompressed += l;
                        e->written_compressed += l;

                        tar_export_report_progress(e);

                        return 0;
                }
        }

        while (e->buffer_size <= 0) {
                uint8_t input[COPY_BUFFER_SIZE];

                if (e->eof) {
                        r = tar_export_finish(e);
                        goto finish;
                }

                l = read(e->tar_fd, input, sizeof(input));
                if (l < 0) {
                        r = log_error_errno(errno, "Failed to read tar file: %m");
                        goto finish;
                }

                if (l == 0) {
                        e->eof = true;
                        r = import_compress_finish(&e->compress, &e->buffer, &e->buffer_size, &e->buffer_allocated);
                } else {
                        e->written_uncompressed += l;
                        r = import_compress(&e->compress, input, l, &e->buffer, &e->buffer_size, &e->buffer_allocated);
                }
                if (r < 0) {
                        r = log_error_errno(r, "Failed to encode: %m");
                        goto finish;
                }
        }

        l = write(e->output_fd, e->buffer, e->buffer_size);
        if (l < 0) {
                if (errno == EAGAIN)
                        return 0;

                r = log_error_errno(errno, "Failed to write output file: %m");
                goto finish;
        }

        assert((size_t) l <= e->buffer_size);
        memmove(e->buffer, (uint8_t*) e->buffer + l, e->buffer_size - l);
        e->buffer_size -= l;
        e->written_compressed += l;

        tar_export_report_progress(e);

        return 0;

finish:
        if (r >= 0 && isatty_safe(STDERR_FILENO))
                clear_progress_bar(/* prefix= */ NULL);

        if (e->on_finished)
                e->on_finished(e, r, e->userdata);
        else
                sd_event_exit(e->event, r);

        return 0;
}

static int tar_export_on_output(sd_event_source *s, int fd, uint32_t revents, void *userdata) {
        TarExport *i = userdata;

        return tar_export_process(i);
}

static int tar_export_on_defer(sd_event_source *s, void *userdata) {
        TarExport *i = userdata;

        return tar_export_process(i);
}

int tar_export_start(TarExport *e, const char *path, int fd, ImportCompressType compress) {
        _cleanup_close_ int sfd = -EBADF;
        int r;

        assert(e);
        assert(path);
        assert(fd >= 0);
        assert(compress < _IMPORT_COMPRESS_TYPE_MAX);
        assert(compress != IMPORT_COMPRESS_UNKNOWN);

        if (e->output_fd >= 0)
                return -EBUSY;

        sfd = open(path, O_DIRECTORY|O_RDONLY|O_NOCTTY|O_CLOEXEC);
        if (sfd < 0)
                return -errno;

        if (fstat(sfd, &e->st) < 0)
                return -errno;

        r = fd_nonblock(fd, true);
        if (r < 0)
                return r;

        r = free_and_strdup(&e->path, path);
        if (r < 0)
                return r;

        e->quota_referenced = UINT64_MAX;

        if (btrfs_might_be_subvol(&e->st)) {
                BtrfsQuotaInfo q;

                r = btrfs_subvol_get_subtree_quota_fd(sfd, 0, &q);
                if (r >= 0)
                        e->quota_referenced = q.referenced;

                e->temp_path = mfree(e->temp_path);

                r = tempfn_random(path, NULL, &e->temp_path);
                if (r < 0)
                        return r;

                /* Let's try to make a snapshot, if we can, so that the export is atomic */
                r = btrfs_subvol_snapshot_at(sfd, NULL, AT_FDCWD, e->temp_path, BTRFS_SNAPSHOT_READ_ONLY|BTRFS_SNAPSHOT_RECURSIVE);
                if (r < 0) {
                        log_debug_errno(r, "Couldn't create snapshot %s of %s, not exporting atomically: %m", e->temp_path, path);
                        e->temp_path = mfree(e->temp_path);
                }
        }

        r = import_compress_init(&e->compress, compress);
        if (r < 0)
                return r;

        r = sd_event_add_io(e->event, &e->output_event_source, fd, EPOLLOUT, tar_export_on_output, e);
        if (r == -EPERM) {
                r = sd_event_add_defer(e->event, &e->output_event_source, tar_export_on_defer, e);
                if (r < 0)
                        return r;

                r = sd_event_source_set_enabled(e->output_event_source, SD_EVENT_ON);
        }
        if (r < 0)
                return r;

        e->tar_fd = import_fork_tar_c(e->temp_path ?: e->path, &e->tar_pid);
        if (e->tar_fd < 0) {
                e->output_event_source = sd_event_source_unref(e->output_event_source);
                return e->tar_fd;
        }

        e->output_fd = fd;
        return r;
}

1	/* SPDX-License-Identifier: LGPL-2.1-or-later */
2
3	#include "sd-daemon.h"
4
5	#include "alloc-util.h"
6	#include "btrfs-util.h"
7	#include "export-tar.h"
8	#include "fd-util.h"
9	#include "import-common.h"
10	#include "log.h"
11	#include "pretty-print.h"
12	#include "process-util.h"
13	#include "ratelimit.h"
14	#include "string-util.h"
15	#include "tmpfile-util.h"
16
17	#define COPY_BUFFER_SIZE (16*1024)
18
19	struct TarExport {
20	sd_event *event;
21
22	TarExportFinished on_finished;
23	void *userdata;
24
25	char *path;
26	char *temp_path;
27
28	int output_fd;
29	int tar_fd;
30
31	ImportCompress compress;
32
33	sd_event_source *output_event_source;
34
35	void *buffer;
36	size_t buffer_size;
37	size_t buffer_allocated;
38
39	uint64_t written_compressed;
40	uint64_t written_uncompressed;
41
42	pid_t tar_pid;
43
44	struct stat st;
45	uint64_t quota_referenced;
46
47	unsigned last_percent;
48	RateLimit progress_ratelimit;
49
50	bool eof;
51	bool tried_splice;
52	};
53
54	TarExport tar_export_unref(TarExport e) {	4✔
55	if (!e)	4✔
56	return NULL;
57
58	sd_event_source_unref(e->output_event_source);	4✔
59
60	if (e->tar_pid > 1)	4✔
61	sigkill_wait(e->tar_pid);	×
62
63	if (e->temp_path) {	4✔
64	(void) btrfs_subvol_remove(e->temp_path, BTRFS_REMOVE_QUOTA);	×
65	free(e->temp_path);	×
66	}
67
68	import_compress_free(&e->compress);	4✔
69
70	sd_event_unref(e->event);	4✔
71
72	safe_close(e->tar_fd);	4✔
73
74	free(e->buffer);	4✔
75	free(e->path);	4✔
76	return mfree(e);	4✔
77	}
78
79	int tar_export_new(	4✔
80	TarExport **ret,
81	sd_event *event,
82	TarExportFinished on_finished,
83	void *userdata) {
84
85	_cleanup_(tar_export_unrefp) TarExport *e = NULL;	4✔
86	int r;	4✔
87
88	assert(ret);	4✔
89
90	e = new(TarExport, 1);	4✔
91	if (!e)	4✔
92	return -ENOMEM;
93
94	*e = (TarExport) {	4✔
95	.output_fd = -EBADF,
96	.tar_fd = -EBADF,
97	.on_finished = on_finished,
98	.userdata = userdata,
99	.quota_referenced = UINT64_MAX,
100	.last_percent = UINT_MAX,
101	.progress_ratelimit = { 100 * USEC_PER_MSEC, 1 },
102	};
103
104	if (event)	4✔
105	e->event = sd_event_ref(event);	4✔
106	else {
107	r = sd_event_default(&e->event);	×
108	if (r < 0)	×
109	return r;
110	}
111
112	*ret = TAKE_PTR(e);	4✔
113
114	return 0;	4✔
115	}
116
117	static void tar_export_report_progress(TarExport *e) {	3,813✔
118	unsigned percent;	3,813✔
119	assert(e);	3,813✔
120
121	/* Do we have any quota info? If not, we don't know anything about the progress */
122	if (e->quota_referenced == UINT64_MAX)	3,813✔
123	return;
124
125	if (e->written_uncompressed >= e->quota_referenced)	×
126	percent = 100;
127	else
128	percent = (unsigned) ((e->written_uncompressed * UINT64_C(100)) / e->quota_referenced);	×
129
130	if (percent == e->last_percent)	×
131	return;
132
133	if (!ratelimit_below(&e->progress_ratelimit))	×
134	return;
135
136	sd_notifyf(false, "X_IMPORT_PROGRESS=%u%%", percent);	×
137
138	if (isatty_safe(STDERR_FILENO))	×
139	(void) draw_progress_barf(	×
140	percent,
141	"%s %s/%s",
142	glyph(GLYPH_ARROW_RIGHT),
143	FORMAT_BYTES(e->written_uncompressed),	×
144	FORMAT_BYTES(e->quota_referenced));	×
145	else
146	log_info("Exported %u%%.", percent);	×
147
148	e->last_percent = percent;	×
149	}
150
151	static int tar_export_finish(TarExport *e) {	4✔
152	int r;	4✔
153
154	assert(e);	4✔
155	assert(e->tar_fd >= 0);	4✔
156
157	if (e->tar_pid > 0) {	4✔
158	r = wait_for_terminate_and_check("tar", TAKE_PID(e->tar_pid), WAIT_LOG);	4✔
159	if (r < 0)	4✔
160	return r;
161	if (r != EXIT_SUCCESS)	4✔
162	return -EPROTO;
163	}
164
165	e->tar_fd = safe_close(e->tar_fd);	4✔
166
167	return 0;	4✔
168	}
169
170	static int tar_export_process(TarExport *e) {	36,891✔
171	ssize_t l;	36,891✔
172	int r;	36,891✔
173
174	assert(e);	36,891✔
175
176	if (!e->tried_splice && e->compress.type == IMPORT_COMPRESS_UNCOMPRESSED) {	36,891✔
177
178	l = splice(e->tar_fd, NULL, e->output_fd, NULL, COPY_BUFFER_SIZE, 0);	33,506✔
179	if (l < 0) {	33,506✔
180	if (errno == EAGAIN)	33,075✔
181	return 0;
182
183	e->tried_splice = true;	1✔
184	} else if (l == 0) {	431✔
185	r = tar_export_finish(e);	×
186	goto finish;	×
187	} else {
188	e->written_uncompressed += l;	431✔
189	e->written_compressed += l;	431✔
190
191	tar_export_report_progress(e);	431✔
192
193	return 0;	431✔
194	}
195	}
196
197	while (e->buffer_size <= 0) {	16,154✔
198	uint8_t input[COPY_BUFFER_SIZE];	12,772✔
199
200	if (e->eof) {	12,772✔
201	r = tar_export_finish(e);	4✔
202	goto finish;	4✔
203	}
204
205	l = read(e->tar_fd, input, sizeof(input));	12,768✔
206	if (l < 0) {	12,768✔
207	r = log_error_errno(errno, "Failed to read tar file: %m");	×
208	goto finish;	×
209	}
210
211	if (l == 0) {	12,768✔
212	e->eof = true;	4✔
213	r = import_compress_finish(&e->compress, &e->buffer, &e->buffer_size, &e->buffer_allocated);	4✔
214	} else {
215	e->written_uncompressed += l;	12,764✔
216	r = import_compress(&e->compress, input, l, &e->buffer, &e->buffer_size, &e->buffer_allocated);	12,764✔
217	}
218	if (r < 0) {	12,768✔
219	r = log_error_errno(r, "Failed to encode: %m");	×
220	goto finish;	×
221	}
222	}
223
224	l = write(e->output_fd, e->buffer, e->buffer_size);	3,382✔
225	if (l < 0) {	3,382✔
226	if (errno == EAGAIN)	×
227	return 0;
228
229	r = log_error_errno(errno, "Failed to write output file: %m");	×
230	goto finish;	×
231	}
232
233	assert((size_t) l <= e->buffer_size);	3,382✔
234	memmove(e->buffer, (uint8_t*) e->buffer + l, e->buffer_size - l);	3,382✔
235	e->buffer_size -= l;	3,382✔
236	e->written_compressed += l;	3,382✔
237
238	tar_export_report_progress(e);	3,382✔
239
240	return 0;	3,382✔
241
242	finish:	4✔
243	if (r >= 0 && isatty_safe(STDERR_FILENO))	4✔
244	clear_progress_bar(/* prefix= */ NULL);	×
245
246	if (e->on_finished)	4✔
247	e->on_finished(e, r, e->userdata);	4✔
248	else
249	sd_event_exit(e->event, r);	×
250
251	return 0;
252	}
253
254	static int tar_export_on_output(sd_event_source s, int fd, uint32_t revents, void userdata) {	33,506✔
255	TarExport *i = userdata;	33,506✔
256
257	return tar_export_process(i);	33,506✔
258	}
259
260	static int tar_export_on_defer(sd_event_source s, void userdata) {	3,385✔
261	TarExport *i = userdata;	3,385✔
262
263	return tar_export_process(i);	3,385✔
264	}
265
266	int tar_export_start(TarExport e, const char path, int fd, ImportCompressType compress) {	4✔
267	_cleanup_close_ int sfd = -EBADF;	4✔
268	int r;	4✔
269
270	assert(e);	4✔
271	assert(path);	4✔
272	assert(fd >= 0);	4✔
273	assert(compress < _IMPORT_COMPRESS_TYPE_MAX);	4✔
274	assert(compress != IMPORT_COMPRESS_UNKNOWN);	4✔
275
276	if (e->output_fd >= 0)	4✔
277	return -EBUSY;
278
279	sfd = open(path, O_DIRECTORY\|O_RDONLY\|O_NOCTTY\|O_CLOEXEC);	4✔
280	if (sfd < 0)	4✔
281	return -errno;	×
282
283	if (fstat(sfd, &e->st) < 0)	4✔
284	return -errno;	×
285
286	r = fd_nonblock(fd, true);	4✔
287	if (r < 0)	4✔
288	return r;
289
290	r = free_and_strdup(&e->path, path);	4✔
291	if (r < 0)	4✔
292	return r;
293
294	e->quota_referenced = UINT64_MAX;	4✔
295
296	if (btrfs_might_be_subvol(&e->st)) {	4✔
297	BtrfsQuotaInfo q;	×
298
299	r = btrfs_subvol_get_subtree_quota_fd(sfd, 0, &q);	×
300	if (r >= 0)	×
301	e->quota_referenced = q.referenced;	×
302
303	e->temp_path = mfree(e->temp_path);	×
304
305	r = tempfn_random(path, NULL, &e->temp_path);	×
306	if (r < 0)	×
307	return r;	×
308
309	/* Let's try to make a snapshot, if we can, so that the export is atomic */
310	r = btrfs_subvol_snapshot_at(sfd, NULL, AT_FDCWD, e->temp_path, BTRFS_SNAPSHOT_READ_ONLY\|BTRFS_SNAPSHOT_RECURSIVE);	×
311	if (r < 0) {	×
312	log_debug_errno(r, "Couldn't create snapshot %s of %s, not exporting atomically: %m", e->temp_path, path);	×
313	e->temp_path = mfree(e->temp_path);	×
314	}
315	}
316
317	r = import_compress_init(&e->compress, compress);	4✔
318	if (r < 0)	4✔
319	return r;
320
321	r = sd_event_add_io(e->event, &e->output_event_source, fd, EPOLLOUT, tar_export_on_output, e);	4✔
322	if (r == -EPERM) {	4✔
323	r = sd_event_add_defer(e->event, &e->output_event_source, tar_export_on_defer, e);	3✔
324	if (r < 0)	3✔
325	return r;
326
327	r = sd_event_source_set_enabled(e->output_event_source, SD_EVENT_ON);	3✔
328	}
329	if (r < 0)	4✔
330	return r;
331
332	e->tar_fd = import_fork_tar_c(e->temp_path ?: e->path, &e->tar_pid);	4✔
333	if (e->tar_fd < 0) {	4✔
334	e->output_event_source = sd_event_source_unref(e->output_event_source);	×
335	return e->tar_fd;	×
336	}
337
338	e->output_fd = fd;	4✔
339	return r;	4✔
340	}

systemd / systemd / 14630481637

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous