15232239991

Committed 24 May 2025 08:01PM UTC coverage: 72.053% (-0.02%) from 72.07%

Build # 15232239991

Build Type

push

github

Committed by

web-flow

Commit Message

docs: add man pages for sd_device_enumerator_[new,ref,unref,unrefp] (#37586)

For #20929.

Run Details

299160 of 415197 relevant lines covered (72.05%)

703671.29 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

58.73

/src/shared/bpf-program.c

/* SPDX-License-Identifier: LGPL-2.1-or-later */

#include <fcntl.h>
#include <linux/bpf.h>
#include <linux/bpf_insn.h>
#include <unistd.h>

#include "alloc-util.h"
#include "bpf-program.h"
#include "errno-util.h"
#include "escape.h"
#include "extract-word.h"
#include "fd-util.h"
#include "fdset.h"
#include "log.h"
#include "memory-util.h"
#include "missing_syscall.h"
#include "parse-util.h"
#include "path-util.h"
#include "serialize.h"
#include "set.h"
#include "string-table.h"
#include "string-util.h"

static const char *const bpf_cgroup_attach_type_table[__MAX_BPF_ATTACH_TYPE] = {
        [BPF_CGROUP_INET_INGRESS] =     "ingress",
        [BPF_CGROUP_INET_EGRESS] =      "egress",
        [BPF_CGROUP_INET_SOCK_CREATE] = "sock_create",
        [BPF_CGROUP_SOCK_OPS] =         "sock_ops",
        [BPF_CGROUP_DEVICE] =           "device",
        [BPF_CGROUP_INET4_BIND] =       "bind4",
        [BPF_CGROUP_INET6_BIND] =       "bind6",
        [BPF_CGROUP_INET4_CONNECT] =    "connect4",
        [BPF_CGROUP_INET6_CONNECT] =    "connect6",
        [BPF_CGROUP_INET4_POST_BIND] =  "post_bind4",
        [BPF_CGROUP_INET6_POST_BIND] =  "post_bind6",
        [BPF_CGROUP_UDP4_SENDMSG] =     "sendmsg4",
        [BPF_CGROUP_UDP6_SENDMSG] =     "sendmsg6",
        [BPF_CGROUP_SYSCTL] =           "sysctl",
        [BPF_CGROUP_UDP4_RECVMSG] =     "recvmsg4",
        [BPF_CGROUP_UDP6_RECVMSG] =     "recvmsg6",
        [BPF_CGROUP_GETSOCKOPT] =       "getsockopt",
        [BPF_CGROUP_SETSOCKOPT] =       "setsockopt",
};

DEFINE_STRING_TABLE_LOOKUP(bpf_cgroup_attach_type, int);

DEFINE_HASH_OPS_WITH_KEY_DESTRUCTOR(bpf_program_hash_ops, void, trivial_hash_func, trivial_compare_func, bpf_program_free);

int bpf_program_supported(void) {
        static int cached = 0;

        if (cached != 0)
                return cached;

        /* Currently, we only use the following three types:
         * - BPF_PROG_TYPE_CGROUP_SKB, supported since kernel v4.10 (0e33661de493db325435d565a4a722120ae4cbf3),
         * - BPF_PROG_TYPE_CGROUP_DEVICE, supported since kernel v4.15 (ebc614f687369f9df99828572b1d85a7c2de3d92),
         * - BPF_PROG_TYPE_CGROUP_SOCK_ADDR, supported since kernel v4.17 (4fbac77d2d092b475dda9eea66da674369665427).
         * Hence, as our baseline on the kernel is v5.4, it is not necessary to check if we can create BPF
         * programs of hthese types.
         *
         * However, unfortunately the kernel allows us to create BPF_PROG_TYPE_CGROUP_SKB (maybe also other types)
         * programs even when CONFIG_CGROUP_BPF is turned off at kernel compilation time. This sucks of course:
         * why does it allow us to create a cgroup BPF program if we can't do a thing with it later?
         *
         * We detect this case by issuing the BPF_PROG_DETACH bpf() call with invalid file descriptors: if
         * CONFIG_CGROUP_BPF is turned off, then the call will fail early with EINVAL. If it is turned on the
         * parameters are validated however, and that'll fail with EBADF then.
         *
         * The check seems also important when we are running with sanitizers. With sanitizers (at least with
         * LLVM v20), the following check and other bpf() calls fails even if the kernel supports BPF. To
         * avoid unexpected fail when running with sanitizers, let's explicitly check if bpf() syscall works. */

        /* Clang and GCC (>=15) do not 0-pad with structured initialization, causing the kernel to reject the
         * bpf_attr as invalid. See: https://github.com/torvalds/linux/blob/v5.9/kernel/bpf/syscall.c#L65
         * Hence, we cannot use structured initialization here, and need to clear the structure with zero
         * explicitly before use. */
        union bpf_attr attr;
        zero(attr);
        attr.attach_type = BPF_CGROUP_INET_EGRESS; /* since kernel v4.10 (0e33661de493db325435d565a4a722120ae4cbf3) */
        attr.target_fd = -EBADF;
        attr.attach_bpf_fd = -EBADF;

        if (bpf(BPF_PROG_DETACH, &attr, sizeof(attr)) < 0) {
                if (errno == EBADF) /* YAY! */
                        return cached = true;

                return cached = log_debug_errno(errno, "Didn't get EBADF from invalid BPF_PROG_DETACH call: %m");
        }

        return cached = log_debug_errno(SYNTHETIC_ERRNO(EBADE),
                                        "Wut? Kernel accepted our invalid BPF_PROG_DETACH call? Something is weird, assuming BPF is broken and hence not supported.");
}

BPFProgram *bpf_program_free(BPFProgram *p) {
        if (!p)
                return NULL;
        /* Unfortunately, the kernel currently doesn't implicitly detach BPF programs from their cgroups when the last
         * fd to the BPF program is closed. This has nasty side-effects since this means that abnormally terminated
         * programs that attached one of their BPF programs to a cgroup will leave this program pinned for good with
         * zero chance of recovery, until the cgroup is removed. This is particularly problematic if the cgroup in
         * question is the root cgroup (or any other cgroup belonging to a service that cannot be restarted during
         * operation, such as dbus), as the memory for the BPF program can only be reclaimed through a reboot. To
         * counter this, we track closely to which cgroup a program was attached to and will detach it on our own
         * whenever we close the BPF fd. */
        (void) bpf_program_cgroup_detach(p);

        safe_close(p->kernel_fd);
        free(p->prog_name);
        free(p->instructions);
        free(p->attached_path);

        return mfree(p);
}

 /* struct bpf_prog_info info must be initialized since its value is both input and output
  * for BPF_OBJ_GET_INFO_BY_FD syscall. */
static int bpf_program_get_info_by_fd(int prog_fd, struct bpf_prog_info *info, uint32_t info_len) {
        union bpf_attr attr;

        /* Explicitly memset to zero since some compilers may produce non-zero-initialized padding when
         * structured initialization is used.
         * Refer to https://github.com/systemd/systemd/issues/18164
         */
        zero(attr);
        attr.info.bpf_fd = prog_fd;
        attr.info.info_len = info_len;
        attr.info.info = PTR_TO_UINT64(info);

        return RET_NERRNO(bpf(BPF_OBJ_GET_INFO_BY_FD, &attr, sizeof(attr)));
}

int bpf_program_new(uint32_t prog_type, const char *prog_name, BPFProgram **ret) {
        _cleanup_(bpf_program_freep) BPFProgram *p = NULL;
        _cleanup_free_ char *name = NULL;

        if (prog_name) {
                if (strlen(prog_name) >= BPF_OBJ_NAME_LEN)
                        return -ENAMETOOLONG;

                name = strdup(prog_name);
                if (!name)
                        return -ENOMEM;
        }

        p = new(BPFProgram, 1);
        if (!p)
                return -ENOMEM;

        *p = (BPFProgram) {
                .prog_type = prog_type,
                .kernel_fd = -EBADF,
                .prog_name = TAKE_PTR(name),
        };

        *ret = TAKE_PTR(p);

        return 0;
}

int bpf_program_new_from_bpffs_path(const char *path, BPFProgram **ret) {
        _cleanup_(bpf_program_freep) BPFProgram *p = NULL;
        struct bpf_prog_info info = {};
        int r;

        assert(path);
        assert(ret);

        p = new(BPFProgram, 1);
        if (!p)
                return -ENOMEM;

        *p = (BPFProgram) {
                .prog_type = BPF_PROG_TYPE_UNSPEC,
                .kernel_fd = -EBADF,
        };

        r = bpf_program_load_from_bpf_fs(p, path);
        if (r < 0)
                return r;

        r = bpf_program_get_info_by_fd(p->kernel_fd, &info, sizeof(info));
        if (r < 0)
                return r;

        p->prog_type = info.type;
        *ret = TAKE_PTR(p);

        return 0;
}

int bpf_program_add_instructions(BPFProgram *p, const struct bpf_insn *instructions, size_t count) {

        assert(p);

        if (p->kernel_fd >= 0) /* don't allow modification after we uploaded things to the kernel */
                return -EBUSY;

        if (!GREEDY_REALLOC(p->instructions, p->n_instructions + count))
                return -ENOMEM;

        memcpy(p->instructions + p->n_instructions, instructions, sizeof(struct bpf_insn) * count);
        p->n_instructions += count;

        return 0;
}

int bpf_program_load_kernel(BPFProgram *p, char *log_buf, size_t log_size) {
        union bpf_attr attr;

        assert(p);

        if (p->kernel_fd >= 0) { /* make this idempotent */
                memzero(log_buf, log_size);
                return 0;
        }

        // FIXME: Clang doesn't 0-pad with structured initialization, causing
        // the kernel to reject the bpf_attr as invalid. See:
        // https://github.com/torvalds/linux/blob/v5.9/kernel/bpf/syscall.c#L65
        // Ideally it should behave like GCC, so that we can remove these workarounds.
        zero(attr);
        attr.prog_type = p->prog_type;
        attr.insns = PTR_TO_UINT64(p->instructions);
        attr.insn_cnt = p->n_instructions;
        attr.license = PTR_TO_UINT64("GPL");
        attr.log_buf = PTR_TO_UINT64(log_buf);
        attr.log_level = !!log_buf;
        attr.log_size = log_size;
        if (p->prog_name)
                strncpy(attr.prog_name, p->prog_name, BPF_OBJ_NAME_LEN - 1);

        p->kernel_fd = bpf(BPF_PROG_LOAD, &attr, sizeof(attr));
        if (p->kernel_fd < 0)
                return -errno;

        return 0;
}

int bpf_program_load_from_bpf_fs(BPFProgram *p, const char *path) {
        union bpf_attr attr;

        assert(p);

        if (p->kernel_fd >= 0) /* don't overwrite an assembled or loaded program */
                return -EBUSY;

        zero(attr);
        attr.pathname = PTR_TO_UINT64(path);

        p->kernel_fd = bpf(BPF_OBJ_GET, &attr, sizeof(attr));
        if (p->kernel_fd < 0)
                return -errno;

        return 0;
}

int bpf_program_cgroup_attach(BPFProgram *p, int type, const char *path, uint32_t flags) {
        _cleanup_free_ char *copy = NULL;
        _cleanup_close_ int fd = -EBADF;
        union bpf_attr attr;
        int r;

        assert(p);
        assert(type >= 0);
        assert(path);

        if (!IN_SET(flags, 0, BPF_F_ALLOW_OVERRIDE, BPF_F_ALLOW_MULTI))
                return -EINVAL;

        /* We need to track which cgroup the program is attached to, and we can only track one attachment, hence let's
        * refuse this early. */
        if (p->attached_path) {
                if (!path_equal(p->attached_path, path))
                        return -EBUSY;
                if (p->attached_type != type)
                        return -EBUSY;
                if (p->attached_flags != flags)
                        return -EBUSY;

                /* Here's a shortcut: if we previously attached this program already, then we don't have to do so
                 * again. Well, with one exception: if we are in BPF_F_ALLOW_OVERRIDE mode then someone else might have
                 * replaced our program since the last time, hence let's reattach it again, just to be safe. In flags
                 * == 0 mode this is not an issue since nobody else can replace our program in that case, and in flags
                 * == BPF_F_ALLOW_MULTI mode any other's program would be installed in addition to ours hence ours
                 * would remain in effect. */
                if (flags != BPF_F_ALLOW_OVERRIDE)
                        return 0;
        }

        /* Ensure we have a kernel object for this. */
        r = bpf_program_load_kernel(p, NULL, 0);
        if (r < 0)
                return r;

        copy = strdup(path);
        if (!copy)
                return -ENOMEM;

        fd = open(path, O_DIRECTORY|O_RDONLY|O_CLOEXEC);
        if (fd < 0)
                return -errno;

        zero(attr);
        attr.attach_type = type;
        attr.target_fd = fd;
        attr.attach_bpf_fd = p->kernel_fd;
        attr.attach_flags = flags;

        if (bpf(BPF_PROG_ATTACH, &attr, sizeof(attr)) < 0)
                return -errno;

        free_and_replace(p->attached_path, copy);
        p->attached_type = type;
        p->attached_flags = flags;

        return 0;
}

int bpf_program_cgroup_detach(BPFProgram *p) {
        _cleanup_close_ int fd = -EBADF;

        assert(p);

        if (!p->attached_path)
                return -EUNATCH;

        fd = open(p->attached_path, O_DIRECTORY|O_RDONLY|O_CLOEXEC);
        if (fd < 0) {
                if (errno != ENOENT)
                        return -errno;

                /* If the cgroup does not exist anymore, then we don't have to explicitly detach, it got detached
                 * implicitly by the removal, hence don't complain */

        } else {
                union bpf_attr attr;

                zero(attr);
                attr.attach_type = p->attached_type;
                attr.target_fd = fd;
                attr.attach_bpf_fd = p->kernel_fd;

                if (bpf(BPF_PROG_DETACH, &attr, sizeof(attr)) < 0)
                        return -errno;
        }

        p->attached_path = mfree(p->attached_path);

        return 0;
}

int bpf_map_new(
                const char *name,
                enum bpf_map_type type,
                size_t key_size,
                size_t value_size,
                size_t max_entries,
                uint32_t flags) {

        union bpf_attr attr;
        const char *n = name;

        zero(attr);
        attr.map_type = type;
        attr.key_size = key_size;
        attr.value_size = value_size;
        attr.max_entries = max_entries;
        attr.map_flags = flags;

        /* The map name is primarily informational for debugging purposes, and typically too short
         * to carry the full unit name, hence we employ a trivial lossy escaping to make it fit
         * (truncation + only alphanumerical, "." and "_" are allowed as per
         * https://docs.kernel.org/bpf/maps.html#usage-notes) */
        for (size_t i = 0; i < sizeof(attr.map_name) - 1 && *n; i++, n++)
                attr.map_name[i] = strchr(ALPHANUMERICAL ".", *n) ? *n : '_';

        return RET_NERRNO(bpf(BPF_MAP_CREATE, &attr, sizeof(attr)));
}

int bpf_map_update_element(int fd, const void *key, void *value) {
        union bpf_attr attr;

        zero(attr);
        attr.map_fd = fd;
        attr.key = PTR_TO_UINT64(key);
        attr.value = PTR_TO_UINT64(value);

        return RET_NERRNO(bpf(BPF_MAP_UPDATE_ELEM, &attr, sizeof(attr)));
}

int bpf_map_lookup_element(int fd, const void *key, void *value) {
        union bpf_attr attr;

        zero(attr);
        attr.map_fd = fd;
        attr.key = PTR_TO_UINT64(key);
        attr.value = PTR_TO_UINT64(value);

        return RET_NERRNO(bpf(BPF_MAP_LOOKUP_ELEM, &attr, sizeof(attr)));
}

int bpf_program_pin(int prog_fd, const char *bpffs_path) {
        union bpf_attr attr;

        zero(attr);
        attr.pathname = PTR_TO_UINT64((void *) bpffs_path);
        attr.bpf_fd = prog_fd;

        return RET_NERRNO(bpf(BPF_OBJ_PIN, &attr, sizeof(attr)));
}

int bpf_program_get_id_by_fd(int prog_fd, uint32_t *ret_id) {
        struct bpf_prog_info info = {};
        int r;

        assert(ret_id);

        r = bpf_program_get_info_by_fd(prog_fd, &info, sizeof(info));
        if (r < 0)
                return r;

        *ret_id = info.id;

        return 0;
};

int bpf_program_serialize_attachment(
                FILE *f,
                FDSet *fds,
                const char *key,
                BPFProgram *p) {

        _cleanup_free_ char *escaped = NULL;
        int copy, r;

        if (!p || !p->attached_path)
                return 0;

        assert(p->kernel_fd >= 0);

        escaped = cescape(p->attached_path);
        if (!escaped)
                return -ENOMEM;

        copy = fdset_put_dup(fds, p->kernel_fd);
        if (copy < 0)
                return log_error_errno(copy, "Failed to add BPF kernel fd to serialize: %m");

        r = serialize_item_format(
                        f,
                        key,
                        "%i %s %s",
                        copy,
                        bpf_cgroup_attach_type_to_string(p->attached_type),
                        escaped);
        if (r < 0)
                return r;

        /* After serialization, let's forget the fact that this program is attached. The attachment — if you
         * so will — is now 'owned' by the serialization, and not us anymore. Why does that matter? Because
         * of BPF's less-than-ideal lifecycle handling: to detach a program from a cgroup we have to
         * explicitly do so, it's not done implicitly on close(). Now, since we are serializing here we don't
         * want the program to be detached while freeing things, so that the attachment can be retained after
         * deserializing again. bpf_program_free() implicitly detaches things, if attached_path is non-NULL,
         * hence we set it to NULL here. */

        p->attached_path = mfree(p->attached_path);
        return 0;
}

int bpf_program_serialize_attachment_set(FILE *f, FDSet *fds, const char *key, Set *set) {
        BPFProgram *p;
        int r;

        SET_FOREACH(p, set) {
                r = bpf_program_serialize_attachment(f, fds, key, p);
                if (r < 0)
                        return r;
        }

        return 0;
}

int bpf_program_deserialize_attachment(const char *v, FDSet *fds, BPFProgram **bpfp) {
        _cleanup_free_ char *sfd = NULL, *sat = NULL, *unescaped = NULL;
        _cleanup_(bpf_program_freep) BPFProgram *p = NULL;
        _cleanup_close_ int fd = -EBADF;
        ssize_t l;
        int ifd, at, r;

        assert(v);
        assert(bpfp);

        /* Extract first word: the fd number */
        r = extract_first_word(&v, &sfd, NULL, 0);
        if (r < 0)
                return r;
        if (r == 0)
                return -EINVAL;

        ifd = parse_fd(sfd);
        if (ifd < 0)
                return r;

        /* Extract second word: the attach type */
        r = extract_first_word(&v, &sat, NULL, 0);
        if (r < 0)
                return r;
        if (r == 0)
                return -EINVAL;

        at = bpf_cgroup_attach_type_from_string(sat);
        if (at < 0)
                return at;

        /* The rest is the path */
        if (isempty(v))
                return -EINVAL;

        l = cunescape(v, 0, &unescaped);
        if (l < 0)
                return l;

        fd = fdset_remove(fds, ifd);
        if (fd < 0)
                return fd;

        p = new(BPFProgram, 1);
        if (!p)
                return -ENOMEM;

        *p = (BPFProgram) {
                .kernel_fd = TAKE_FD(fd),
                .prog_type = BPF_PROG_TYPE_UNSPEC,
                .attached_path = TAKE_PTR(unescaped),
                .attached_type = at,
        };

        if (*bpfp)
                bpf_program_free(*bpfp);

        *bpfp = TAKE_PTR(p);
        return 0;
}

int bpf_program_deserialize_attachment_set(const char *v, FDSet *fds, Set **bpfsetp) {
        BPFProgram *p = NULL;
        int r;

        assert(v);
        assert(bpfsetp);

        r = bpf_program_deserialize_attachment(v, fds, &p);
        if (r < 0)
                return r;

        r = set_ensure_consume(bpfsetp, &bpf_program_hash_ops, p);
        if (r < 0)
                return r;

        return 0;
}

1	/* SPDX-License-Identifier: LGPL-2.1-or-later */
2
3	#include <fcntl.h>
4	#include <linux/bpf.h>
5	#include <linux/bpf_insn.h>
6	#include <unistd.h>
7
8	#include "alloc-util.h"
9	#include "bpf-program.h"
10	#include "errno-util.h"
11	#include "escape.h"
12	#include "extract-word.h"
13	#include "fd-util.h"
14	#include "fdset.h"
15	#include "log.h"
16	#include "memory-util.h"
17	#include "missing_syscall.h"
18	#include "parse-util.h"
19	#include "path-util.h"
20	#include "serialize.h"
21	#include "set.h"
22	#include "string-table.h"
23	#include "string-util.h"
24
25	static const char *const bpf_cgroup_attach_type_table[__MAX_BPF_ATTACH_TYPE] = {
26	[BPF_CGROUP_INET_INGRESS] = "ingress",
27	[BPF_CGROUP_INET_EGRESS] = "egress",
28	[BPF_CGROUP_INET_SOCK_CREATE] = "sock_create",
29	[BPF_CGROUP_SOCK_OPS] = "sock_ops",
30	[BPF_CGROUP_DEVICE] = "device",
31	[BPF_CGROUP_INET4_BIND] = "bind4",
32	[BPF_CGROUP_INET6_BIND] = "bind6",
33	[BPF_CGROUP_INET4_CONNECT] = "connect4",
34	[BPF_CGROUP_INET6_CONNECT] = "connect6",
35	[BPF_CGROUP_INET4_POST_BIND] = "post_bind4",
36	[BPF_CGROUP_INET6_POST_BIND] = "post_bind6",
37	[BPF_CGROUP_UDP4_SENDMSG] = "sendmsg4",
38	[BPF_CGROUP_UDP6_SENDMSG] = "sendmsg6",
39	[BPF_CGROUP_SYSCTL] = "sysctl",
40	[BPF_CGROUP_UDP4_RECVMSG] = "recvmsg4",
41	[BPF_CGROUP_UDP6_RECVMSG] = "recvmsg6",
42	[BPF_CGROUP_GETSOCKOPT] = "getsockopt",
43	[BPF_CGROUP_SETSOCKOPT] = "setsockopt",
44	};
45
46	DEFINE_STRING_TABLE_LOOKUP(bpf_cgroup_attach_type, int);	738✔
47
48	DEFINE_HASH_OPS_WITH_KEY_DESTRUCTOR(bpf_program_hash_ops, void, trivial_hash_func, trivial_compare_func, bpf_program_free);	×
49
50	int bpf_program_supported(void) {	2,167✔
51	static int cached = 0;	2,167✔
52
53	if (cached != 0)	2,167✔
54	return cached;	2,167✔
55
56	/* Currently, we only use the following three types:
57	* - BPF_PROG_TYPE_CGROUP_SKB, supported since kernel v4.10 (0e33661de493db325435d565a4a722120ae4cbf3),
58	* - BPF_PROG_TYPE_CGROUP_DEVICE, supported since kernel v4.15 (ebc614f687369f9df99828572b1d85a7c2de3d92),
59	* - BPF_PROG_TYPE_CGROUP_SOCK_ADDR, supported since kernel v4.17 (4fbac77d2d092b475dda9eea66da674369665427).
60	* Hence, as our baseline on the kernel is v5.4, it is not necessary to check if we can create BPF
61	* programs of hthese types.
62	*
63	* However, unfortunately the kernel allows us to create BPF_PROG_TYPE_CGROUP_SKB (maybe also other types)
64	* programs even when CONFIG_CGROUP_BPF is turned off at kernel compilation time. This sucks of course:
65	* why does it allow us to create a cgroup BPF program if we can't do a thing with it later?
66	*
67	* We detect this case by issuing the BPF_PROG_DETACH bpf() call with invalid file descriptors: if
68	* CONFIG_CGROUP_BPF is turned off, then the call will fail early with EINVAL. If it is turned on the
69	* parameters are validated however, and that'll fail with EBADF then.
70	*
71	* The check seems also important when we are running with sanitizers. With sanitizers (at least with
72	* LLVM v20), the following check and other bpf() calls fails even if the kernel supports BPF. To
73	* avoid unexpected fail when running with sanitizers, let's explicitly check if bpf() syscall works. */
74
75	/* Clang and GCC (>=15) do not 0-pad with structured initialization, causing the kernel to reject the
76	* bpf_attr as invalid. See: https://github.com/torvalds/linux/blob/v5.9/kernel/bpf/syscall.c#L65
77	* Hence, we cannot use structured initialization here, and need to clear the structure with zero
78	* explicitly before use. */
79	union bpf_attr attr;	245✔
80	zero(attr);	245✔
81	attr.attach_type = BPF_CGROUP_INET_EGRESS; /* since kernel v4.10 (0e33661de493db325435d565a4a722120ae4cbf3) */	245✔
82	attr.target_fd = -EBADF;	245✔
83	attr.attach_bpf_fd = -EBADF;	245✔
84
85	if (bpf(BPF_PROG_DETACH, &attr, sizeof(attr)) < 0) {	245✔
86	if (errno == EBADF) /* YAY! */	245✔
87	return cached = true;	104✔
88
89	return cached = log_debug_errno(errno, "Didn't get EBADF from invalid BPF_PROG_DETACH call: %m");	141✔
90	}
91
92	return cached = log_debug_errno(SYNTHETIC_ERRNO(EBADE),	×
93	"Wut? Kernel accepted our invalid BPF_PROG_DETACH call? Something is weird, assuming BPF is broken and hence not supported.");
94	}
95
96	BPFProgram bpf_program_free(BPFProgram p) {	117,437✔
97	if (!p)	117,437✔
98	return NULL;
99	/* Unfortunately, the kernel currently doesn't implicitly detach BPF programs from their cgroups when the last
100	* fd to the BPF program is closed. This has nasty side-effects since this means that abnormally terminated
101	* programs that attached one of their BPF programs to a cgroup will leave this program pinned for good with
102	* zero chance of recovery, until the cgroup is removed. This is particularly problematic if the cgroup in
103	* question is the root cgroup (or any other cgroup belonging to a service that cannot be restarted during
104	* operation, such as dbus), as the memory for the BPF program can only be reclaimed through a reboot. To
105	* counter this, we track closely to which cgroup a program was attached to and will detach it on our own
106	* whenever we close the BPF fd. */
107	(void) bpf_program_cgroup_detach(p);	651✔
108
109	safe_close(p->kernel_fd);	651✔
110	free(p->prog_name);	651✔
111	free(p->instructions);	651✔
112	free(p->attached_path);	651✔
113
114	return mfree(p);	651✔
115	}
116
117	/* struct bpf_prog_info info must be initialized since its value is both input and output
118	* for BPF_OBJ_GET_INFO_BY_FD syscall. */
119	static int bpf_program_get_info_by_fd(int prog_fd, struct bpf_prog_info *info, uint32_t info_len) {	×
120	union bpf_attr attr;	×
121
122	/* Explicitly memset to zero since some compilers may produce non-zero-initialized padding when
123	* structured initialization is used.
124	* Refer to https://github.com/systemd/systemd/issues/18164
125	*/
126	zero(attr);	×
127	attr.info.bpf_fd = prog_fd;	×
128	attr.info.info_len = info_len;	×
129	attr.info.info = PTR_TO_UINT64(info);	×
130
131	return RET_NERRNO(bpf(BPF_OBJ_GET_INFO_BY_FD, &attr, sizeof(attr)));	×
132	}
133
134	int bpf_program_new(uint32_t prog_type, const char prog_name, BPFProgram *ret) {	338✔
135	_cleanup_(bpf_program_freep) BPFProgram *p = NULL;	676✔
136	_cleanup_free_ char *name = NULL;	338✔
137
138	if (prog_name) {	338✔
139	if (strlen(prog_name) >= BPF_OBJ_NAME_LEN)	338✔
140	return -ENAMETOOLONG;
141
142	name = strdup(prog_name);	338✔
143	if (!name)	338✔
144	return -ENOMEM;
145	}
146
147	p = new(BPFProgram, 1);	338✔
148	if (!p)	338✔
149	return -ENOMEM;
150
151	*p = (BPFProgram) {	338✔
152	.prog_type = prog_type,
153	.kernel_fd = -EBADF,
154	.prog_name = TAKE_PTR(name),	338✔
155	};
156
157	*ret = TAKE_PTR(p);	338✔
158
159	return 0;	338✔
160	}
161
162	int bpf_program_new_from_bpffs_path(const char path, BPFProgram *ret) {	×
163	_cleanup_(bpf_program_freep) BPFProgram *p = NULL;	×
164	struct bpf_prog_info info = {};	×
165	int r;	×
166
167	assert(path);	×
168	assert(ret);	×
169
170	p = new(BPFProgram, 1);	×
171	if (!p)	×
172	return -ENOMEM;
173
174	*p = (BPFProgram) {	×
175	.prog_type = BPF_PROG_TYPE_UNSPEC,
176	.kernel_fd = -EBADF,
177	};
178
179	r = bpf_program_load_from_bpf_fs(p, path);	×
180	if (r < 0)	×
181	return r;
182
183	r = bpf_program_get_info_by_fd(p->kernel_fd, &info, sizeof(info));	×
184	if (r < 0)	×
185	return r;
186
187	p->prog_type = info.type;	×
188	*ret = TAKE_PTR(p);	×
189
190	return 0;	×
191	}
192
193	int bpf_program_add_instructions(BPFProgram p, const struct bpf_insn instructions, size_t count) {	2,989✔
194
195	assert(p);	2,989✔
196
197	if (p->kernel_fd >= 0) /* don't allow modification after we uploaded things to the kernel */	2,989✔
198	return -EBUSY;
199
200	if (!GREEDY_REALLOC(p->instructions, p->n_instructions + count))	2,989✔
201	return -ENOMEM;
202
203	memcpy(p->instructions + p->n_instructions, instructions, sizeof(struct bpf_insn) * count);	2,989✔
204	p->n_instructions += count;	2,989✔
205
206	return 0;	2,989✔
207	}
208
209	int bpf_program_load_kernel(BPFProgram p, char log_buf, size_t log_size) {	338✔
210	union bpf_attr attr;	338✔
211
212	assert(p);	338✔
213
214	if (p->kernel_fd >= 0) { /* make this idempotent */	338✔
215	memzero(log_buf, log_size);	×
216	return 0;	×
217	}
218
219	// FIXME: Clang doesn't 0-pad with structured initialization, causing
220	// the kernel to reject the bpf_attr as invalid. See:
221	// https://github.com/torvalds/linux/blob/v5.9/kernel/bpf/syscall.c#L65
222	// Ideally it should behave like GCC, so that we can remove these workarounds.
223	zero(attr);	338✔
224	attr.prog_type = p->prog_type;	338✔
225	attr.insns = PTR_TO_UINT64(p->instructions);	338✔
226	attr.insn_cnt = p->n_instructions;	338✔
227	attr.license = PTR_TO_UINT64("GPL");	338✔
228	attr.log_buf = PTR_TO_UINT64(log_buf);	338✔
229	attr.log_level = !!log_buf;	338✔
230	attr.log_size = log_size;	338✔
231	if (p->prog_name)	338✔
232	strncpy(attr.prog_name, p->prog_name, BPF_OBJ_NAME_LEN - 1);	338✔
233
234	p->kernel_fd = bpf(BPF_PROG_LOAD, &attr, sizeof(attr));	338✔
235	if (p->kernel_fd < 0)	338✔
236	return -errno;	2✔
237
238	return 0;
239	}
240
241	int bpf_program_load_from_bpf_fs(BPFProgram p, const char path) {	×
242	union bpf_attr attr;	×
243
244	assert(p);	×
245
246	if (p->kernel_fd >= 0) /* don't overwrite an assembled or loaded program */	×
247	return -EBUSY;	×
248
249	zero(attr);	×
250	attr.pathname = PTR_TO_UINT64(path);	×
251
252	p->kernel_fd = bpf(BPF_OBJ_GET, &attr, sizeof(attr));	×
253	if (p->kernel_fd < 0)	×
254	return -errno;	×
255
256	return 0;
257	}
258
259	int bpf_program_cgroup_attach(BPFProgram p, int type, const char path, uint32_t flags) {	338✔
260	_cleanup_free_ char *copy = NULL;	338✔
261	_cleanup_close_ int fd = -EBADF;	338✔
262	union bpf_attr attr;	338✔
263	int r;	338✔
264
265	assert(p);	338✔
266	assert(type >= 0);	338✔
267	assert(path);	338✔
268
269	if (!IN_SET(flags, 0, BPF_F_ALLOW_OVERRIDE, BPF_F_ALLOW_MULTI))	338✔
270	return -EINVAL;
271
272	/* We need to track which cgroup the program is attached to, and we can only track one attachment, hence let's
273	* refuse this early. */
274	if (p->attached_path) {	338✔
275	if (!path_equal(p->attached_path, path))	×
276	return -EBUSY;
277	if (p->attached_type != type)	×
278	return -EBUSY;
279	if (p->attached_flags != flags)	×
280	return -EBUSY;
281
282	/* Here's a shortcut: if we previously attached this program already, then we don't have to do so
283	* again. Well, with one exception: if we are in BPF_F_ALLOW_OVERRIDE mode then someone else might have
284	* replaced our program since the last time, hence let's reattach it again, just to be safe. In flags
285	* == 0 mode this is not an issue since nobody else can replace our program in that case, and in flags
286	* == BPF_F_ALLOW_MULTI mode any other's program would be installed in addition to ours hence ours
287	* would remain in effect. */
288	if (flags != BPF_F_ALLOW_OVERRIDE)	×
289	return 0;
290	}
291
292	/* Ensure we have a kernel object for this. */
293	r = bpf_program_load_kernel(p, NULL, 0);	338✔
294	if (r < 0)	338✔
295	return r;
296
297	copy = strdup(path);	336✔
298	if (!copy)	336✔
299	return -ENOMEM;
300
301	fd = open(path, O_DIRECTORY\|O_RDONLY\|O_CLOEXEC);	336✔
302	if (fd < 0)	336✔
303	return -errno;	×
304
305	zero(attr);	336✔
306	attr.attach_type = type;	336✔
307	attr.target_fd = fd;	336✔
308	attr.attach_bpf_fd = p->kernel_fd;	336✔
309	attr.attach_flags = flags;	336✔
310
311	if (bpf(BPF_PROG_ATTACH, &attr, sizeof(attr)) < 0)	336✔
312	return -errno;	×
313
314	free_and_replace(p->attached_path, copy);	336✔
315	p->attached_type = type;	336✔
316	p->attached_flags = flags;	336✔
317
318	return 0;	336✔
319	}
320
321	int bpf_program_cgroup_detach(BPFProgram *p) {	651✔
322	_cleanup_close_ int fd = -EBADF;	651✔
323
324	assert(p);	651✔
325
326	if (!p->attached_path)	651✔
327	return -EUNATCH;
328
329	fd = open(p->attached_path, O_DIRECTORY\|O_RDONLY\|O_CLOEXEC);	224✔
330	if (fd < 0) {	224✔
331	if (errno != ENOENT)	105✔
332	return -errno;	×
333
334	/* If the cgroup does not exist anymore, then we don't have to explicitly detach, it got detached
335	* implicitly by the removal, hence don't complain */
336
337	} else {
338	union bpf_attr attr;	119✔
339
340	zero(attr);	119✔
341	attr.attach_type = p->attached_type;	119✔
342	attr.target_fd = fd;	119✔
343	attr.attach_bpf_fd = p->kernel_fd;	119✔
344
345	if (bpf(BPF_PROG_DETACH, &attr, sizeof(attr)) < 0)	119✔
346	return -errno;	×
347	}
348
349	p->attached_path = mfree(p->attached_path);	224✔
350
351	return 0;	224✔
352	}
353
354	int bpf_map_new(	×
355	const char *name,
356	enum bpf_map_type type,
357	size_t key_size,
358	size_t value_size,
359	size_t max_entries,
360	uint32_t flags) {
361
362	union bpf_attr attr;	×
363	const char *n = name;	×
364
365	zero(attr);	×
366	attr.map_type = type;	×
367	attr.key_size = key_size;	×
368	attr.value_size = value_size;	×
369	attr.max_entries = max_entries;	×
370	attr.map_flags = flags;	×
371
372	/* The map name is primarily informational for debugging purposes, and typically too short
373	* to carry the full unit name, hence we employ a trivial lossy escaping to make it fit
374	* (truncation + only alphanumerical, "." and "_" are allowed as per
375	* https://docs.kernel.org/bpf/maps.html#usage-notes) */
376	for (size_t i = 0; i < sizeof(attr.map_name) - 1 && *n; i++, n++)	×
377	attr.map_name[i] = strchr(ALPHANUMERICAL ".", n) ? n : '_';	×
378
379	return RET_NERRNO(bpf(BPF_MAP_CREATE, &attr, sizeof(attr)));	×
380	}
381
382	int bpf_map_update_element(int fd, const void key, void value) {	×
383	union bpf_attr attr;	×
384
385	zero(attr);	×
386	attr.map_fd = fd;	×
387	attr.key = PTR_TO_UINT64(key);	×
388	attr.value = PTR_TO_UINT64(value);	×
389
390	return RET_NERRNO(bpf(BPF_MAP_UPDATE_ELEM, &attr, sizeof(attr)));	×
391	}
392
393	int bpf_map_lookup_element(int fd, const void key, void value) {	×
394	union bpf_attr attr;	×
395
396	zero(attr);	×
397	attr.map_fd = fd;	×
398	attr.key = PTR_TO_UINT64(key);	×
399	attr.value = PTR_TO_UINT64(value);	×
400
401	return RET_NERRNO(bpf(BPF_MAP_LOOKUP_ELEM, &attr, sizeof(attr)));	×
402	}
403
404	int bpf_program_pin(int prog_fd, const char *bpffs_path) {	×
405	union bpf_attr attr;	×
406
407	zero(attr);	×
408	attr.pathname = PTR_TO_UINT64((void *) bpffs_path);	×
409	attr.bpf_fd = prog_fd;	×
410
411	return RET_NERRNO(bpf(BPF_OBJ_PIN, &attr, sizeof(attr)));	×
412	}
413
414	int bpf_program_get_id_by_fd(int prog_fd, uint32_t *ret_id) {	×
415	struct bpf_prog_info info = {};	×
416	int r;	×
417
418	assert(ret_id);	×
419
420	r = bpf_program_get_info_by_fd(prog_fd, &info, sizeof(info));	×
421	if (r < 0)	×
422	return r;	×
423
424	*ret_id = info.id;	×
425
426	return 0;	×
427	};
428
429	int bpf_program_serialize_attachment(	15,690✔
430	FILE *f,
431	FDSet *fds,
432	const char *key,
433	BPFProgram *p) {
434
435	_cleanup_free_ char *escaped = NULL;	15,690✔
436	int copy, r;	15,690✔
437
438	if (!p \|\| !p->attached_path)	15,690✔
439	return 0;
440
441	assert(p->kernel_fd >= 0);	425✔
442
443	escaped = cescape(p->attached_path);	425✔
444	if (!escaped)	425✔
445	return -ENOMEM;
446
447	copy = fdset_put_dup(fds, p->kernel_fd);	425✔
448	if (copy < 0)	425✔
449	return log_error_errno(copy, "Failed to add BPF kernel fd to serialize: %m");	×
450
451	r = serialize_item_format(	425✔
452	f,
453	key,
454	"%i %s %s",
455	copy,
456	bpf_cgroup_attach_type_to_string(p->attached_type),
457	escaped);
458	if (r < 0)	425✔
459	return r;
460
461	/* After serialization, let's forget the fact that this program is attached. The attachment — if you
462	* so will — is now 'owned' by the serialization, and not us anymore. Why does that matter? Because
463	* of BPF's less-than-ideal lifecycle handling: to detach a program from a cgroup we have to
464	* explicitly do so, it's not done implicitly on close(). Now, since we are serializing here we don't
465	* want the program to be detached while freeing things, so that the attachment can be retained after
466	* deserializing again. bpf_program_free() implicitly detaches things, if attached_path is non-NULL,
467	* hence we set it to NULL here. */
468
469	p->attached_path = mfree(p->attached_path);	425✔
470	return 0;	425✔
471	}
472
473	int bpf_program_serialize_attachment_set(FILE f, FDSet fds, const char key, Set set) {	10,460✔
474	BPFProgram *p;	10,460✔
475	int r;	10,460✔
476
477	SET_FOREACH(p, set) {	10,460✔
478	r = bpf_program_serialize_attachment(f, fds, key, p);	×
479	if (r < 0)	×
480	return r;	×
481	}
482
483	return 0;	10,460✔
484	}
485
486	int bpf_program_deserialize_attachment(const char v, FDSet fds, BPFProgram **bpfp) {	313✔
487	_cleanup_free_ char sfd = NULL, sat = NULL, *unescaped = NULL;	313✔
488	_cleanup_(bpf_program_freep) BPFProgram *p = NULL;	×
489	_cleanup_close_ int fd = -EBADF;	313✔
490	ssize_t l;	313✔
491	int ifd, at, r;	313✔
492
493	assert(v);	313✔
494	assert(bpfp);	313✔
495
496	/* Extract first word: the fd number */
497	r = extract_first_word(&v, &sfd, NULL, 0);	313✔
498	if (r < 0)	313✔
499	return r;
500	if (r == 0)	313✔
501	return -EINVAL;
502
503	ifd = parse_fd(sfd);	313✔
504	if (ifd < 0)	313✔
505	return r;
506
507	/* Extract second word: the attach type */
508	r = extract_first_word(&v, &sat, NULL, 0);	313✔
509	if (r < 0)	313✔
510	return r;
511	if (r == 0)	313✔
512	return -EINVAL;
513
514	at = bpf_cgroup_attach_type_from_string(sat);	313✔
515	if (at < 0)	313✔
516	return at;
517
518	/* The rest is the path */
519	if (isempty(v))	626✔
520	return -EINVAL;
521
522	l = cunescape(v, 0, &unescaped);	313✔
523	if (l < 0)	313✔
524	return l;	×
525
526	fd = fdset_remove(fds, ifd);	313✔
527	if (fd < 0)	313✔
528	return fd;
529
530	p = new(BPFProgram, 1);	313✔
531	if (!p)	313✔
532	return -ENOMEM;
533
534	*p = (BPFProgram) {	313✔
535	.kernel_fd = TAKE_FD(fd),	313✔
536	.prog_type = BPF_PROG_TYPE_UNSPEC,
537	.attached_path = TAKE_PTR(unescaped),	313✔
538	.attached_type = at,
539	};
540
541	if (*bpfp)	313✔
542	bpf_program_free(*bpfp);	×
543
544	*bpfp = TAKE_PTR(p);	313✔
545	return 0;	313✔
546	}
547
548	int bpf_program_deserialize_attachment_set(const char v, FDSet fds, Set **bpfsetp) {	×
549	BPFProgram *p = NULL;	×
550	int r;	×
551
552	assert(v);	×
553	assert(bpfsetp);	×
554
555	r = bpf_program_deserialize_attachment(v, fds, &p);	×
556	if (r < 0)	×
557	return r;	×
558
559	r = set_ensure_consume(bpfsetp, &bpf_program_hash_ops, p);	×
560	if (r < 0)	×
561	return r;	×
562
563	return 0;
564	}

systemd / systemd / 15232239991

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous