13380515387

Committed 17 Feb 2025 09:20PM UTC coverage: 71.822% (+0.1%) from 71.714%

Build # 13380515387

Build Type

push

github

Committed by

DaanDeMeyer

Commit Message

ukify: print all remaining log-like output to stderr

We want to be able to capture stdout for json and such, so convert
all remaining logging to stderr.

Coverage Stats

293883 of 409184 relevant lines covered (71.82%)

716959.33 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

95.51

/src/basic/utf8.c

/* SPDX-License-Identifier: LGPL-2.0-or-later */

/* Parts of this file are based on the GLIB utf8 validation functions. The original copyright follows.
 *
 * gutf8.c - Operations on UTF-8 strings.
 * Copyright (C) 1999 Tom Tromey
 * Copyright (C) 2000 Red Hat, Inc.
 */

#include <errno.h>
#include <stdbool.h>
#include <stdlib.h>

#include "alloc-util.h"
#include "gunicode.h"
#include "hexdecoct.h"
#include "macro.h"
#include "string-util.h"
#include "utf8.h"

bool unichar_is_valid(char32_t ch) {

        if (ch >= 0x110000) /* End of unicode space */
                return false;
        if ((ch & 0xFFFFF800) == 0xD800) /* Reserved area for UTF-16 */
                return false;
        if ((ch >= 0xFDD0) && (ch <= 0xFDEF)) /* Reserved */
                return false;
        if ((ch & 0xFFFE) == 0xFFFE) /* BOM (Byte Order Mark) */
                return false;

        return true;
}

static bool unichar_is_control(char32_t ch) {

        /*
          0 to ' '-1 is the C0 range.
          DEL=0x7F, and DEL+1 to 0x9F is C1 range.
          '\t' is in C0 range, but more or less harmless and commonly used.
        */

        return (ch < ' ' && !IN_SET(ch, '\t', '\n')) ||
                (0x7F <= ch && ch <= 0x9F);
}

/* count of characters used to encode one unicode char */
static size_t utf8_encoded_expected_len(uint8_t c) {
        if (c < 0x80)
                return 1;
        if ((c & 0xe0) == 0xc0)
                return 2;
        if ((c & 0xf0) == 0xe0)
                return 3;
        if ((c & 0xf8) == 0xf0)
                return 4;
        if ((c & 0xfc) == 0xf8)
                return 5;
        if ((c & 0xfe) == 0xfc)
                return 6;

        return 0;
}

/* decode one unicode char */
int utf8_encoded_to_unichar(const char *str, char32_t *ret_unichar) {
        char32_t unichar;
        size_t len;

        assert(str);

        len = utf8_encoded_expected_len(str[0]);

        switch (len) {
        case 1:
                *ret_unichar = (char32_t)str[0];
                return 1;
        case 2:
                unichar = str[0] & 0x1f;
                break;
        case 3:
                unichar = (char32_t)str[0] & 0x0f;
                break;
        case 4:
                unichar = (char32_t)str[0] & 0x07;
                break;
        case 5:
                unichar = (char32_t)str[0] & 0x03;
                break;
        case 6:
                unichar = (char32_t)str[0] & 0x01;
                break;
        default:
                return -EINVAL;
        }

        for (size_t i = 1; i < len; i++) {
                if (((char32_t)str[i] & 0xc0) != 0x80)
                        return -EINVAL;

                unichar <<= 6;
                unichar |= (char32_t)str[i] & 0x3f;
        }

        *ret_unichar = unichar;
        return len;
}

bool utf8_is_printable_newline(const char* str, size_t length, bool allow_newline) {
        assert(str);

        for (const char *p = str; length > 0;) {
                int encoded_len;
                char32_t val;

                encoded_len = utf8_encoded_valid_unichar(p, length);
                if (encoded_len < 0)
                        return false;
                assert(encoded_len > 0 && (size_t) encoded_len <= length);

                if (utf8_encoded_to_unichar(p, &val) < 0 ||
                    unichar_is_control(val) ||
                    (!allow_newline && val == '\n'))
                        return false;

                length -= encoded_len;
                p += encoded_len;
        }

        return true;
}

char* utf8_is_valid_n(const char *str, size_t len_bytes) {
        /* Check if the string is composed of valid utf8 characters. If length len_bytes is given, stop after
         * len_bytes. Otherwise, stop at NUL. */

        assert(str);

        for (size_t i = 0; len_bytes != SIZE_MAX ? i < len_bytes : str[i] != '\0'; ) {
                int len;

                if (_unlikely_(str[i] == '\0'))
                        return NULL; /* embedded NUL */

                len = utf8_encoded_valid_unichar(str + i,
                                                 len_bytes != SIZE_MAX ? len_bytes - i : SIZE_MAX);
                if (_unlikely_(len < 0))
                        return NULL; /* invalid character */

                i += len;
        }

        return (char*) str;
}

char* utf8_escape_invalid(const char *str) {
        char *p, *s;

        assert(str);

        p = s = malloc(strlen(str) * 4 + 1);
        if (!p)
                return NULL;

        while (*str) {
                int len;

                len = utf8_encoded_valid_unichar(str, SIZE_MAX);
                if (len > 0) {
                        s = mempcpy(s, str, len);
                        str += len;
                } else {
                        s = stpcpy(s, UTF8_REPLACEMENT_CHARACTER);
                        str += 1;
                }
        }

        *s = '\0';
        return str_realloc(p);
}

int utf8_char_console_width(const char *str) {
        char32_t c;
        int r;

        r = utf8_encoded_to_unichar(str, &c);
        if (r < 0)
                return r;

        if (c == '\t')
                return 8; /* Assume a tab width of 8 */

        /* TODO: we should detect combining characters */

        return unichar_iswide(c) ? 2 : 1;
}

char* utf8_escape_non_printable_full(const char *str, size_t console_width, bool force_ellipsis) {
        char *p, *s, *prev_s;
        size_t n = 0; /* estimated print width */

        assert(str);

        if (console_width == 0)
                return strdup("");

        p = s = prev_s = malloc(strlen(str) * 4 + 1);
        if (!p)
                return NULL;

        for (;;) {
                int len;
                char *saved_s = s;

                if (!*str) { /* done! */
                        if (force_ellipsis)
                                goto truncation;
                        else
                                goto finish;
                }

                len = utf8_encoded_valid_unichar(str, SIZE_MAX);
                if (len > 0) {
                        if (utf8_is_printable(str, len)) {
                                int w;

                                w = utf8_char_console_width(str);
                                assert(w >= 0);
                                if (n + w > console_width)
                                        goto truncation;

                                s = mempcpy(s, str, len);
                                str += len;
                                n += w;

                        } else {
                                for (; len > 0; len--) {
                                        if (n + 4 > console_width)
                                                goto truncation;

                                        *(s++) = '\\';
                                        *(s++) = 'x';
                                        *(s++) = hexchar((int) *str >> 4);
                                        *(s++) = hexchar((int) *str);

                                        str += 1;
                                        n += 4;
                                }
                        }
                } else {
                        if (n + 1 > console_width)
                                goto truncation;

                        s = mempcpy(s, UTF8_REPLACEMENT_CHARACTER, strlen(UTF8_REPLACEMENT_CHARACTER));
                        str += 1;
                        n += 1;
                }

                prev_s = saved_s;
        }

 truncation:
        /* Try to go back one if we don't have enough space for the ellipsis */
        if (n + 1 > console_width)
                s = prev_s;

        s = mempcpy(s, "…", strlen("…"));

 finish:
        *s = '\0';
        return str_realloc(p);
}

char* ascii_is_valid_n(const char *str, size_t len) {
        /* Check whether the string consists of valid ASCII bytes, i.e values between 1 and 127, inclusive.
         * Stops at len, or NUL byte if len is SIZE_MAX. */

        assert(str);

        for (size_t i = 0; len != SIZE_MAX ? i < len : str[i] != '\0'; i++)
                if ((unsigned char) str[i] >= 128 || str[i] == '\0')
                        return NULL;

        return (char*) str;
}

int utf8_to_ascii(const char *str, char replacement_char, char **ret) {
        /* Convert to a string that has only ASCII chars, replacing anything that is not ASCII
         * by replacement_char. */

        _cleanup_free_ char *ans = new(char, strlen(str) + 1);
        if (!ans)
                return -ENOMEM;

        char *q = ans;

        for (const char *p = str; *p; q++) {
                int l;

                l = utf8_encoded_valid_unichar(p, SIZE_MAX);
                if (l < 0)  /* Non-UTF-8, let's not even try to propagate the garbage */
                        return l;

                if (l == 1)
                        *q = *p;
                else
                        /* non-ASCII, we need to replace it */
                        *q = replacement_char;

                p += l;
        }
        *q = '\0';

        *ret = TAKE_PTR(ans);
        return 0;
}

/**
 * utf8_encode_unichar() - Encode single UCS-4 character as UTF-8
 * @out_utf8: output buffer of at least 4 bytes or NULL
 * @g: UCS-4 character to encode
 *
 * This encodes a single UCS-4 character as UTF-8 and writes it into @out_utf8.
 * The length of the character is returned. It is not zero-terminated! If the
 * output buffer is NULL, only the length is returned.
 *
 * Returns: The length in bytes that the UTF-8 representation does or would
 *          occupy.
 */
size_t utf8_encode_unichar(char *out_utf8, char32_t g) {

        if (g < (1 << 7)) {
                if (out_utf8)
                        out_utf8[0] = g & 0x7f;
                return 1;
        } else if (g < (1 << 11)) {
                if (out_utf8) {
                        out_utf8[0] = 0xc0 | ((g >> 6) & 0x1f);
                        out_utf8[1] = 0x80 | (g & 0x3f);
                }
                return 2;
        } else if (g < (1 << 16)) {
                if (out_utf8) {
                        out_utf8[0] = 0xe0 | ((g >> 12) & 0x0f);
                        out_utf8[1] = 0x80 | ((g >> 6) & 0x3f);
                        out_utf8[2] = 0x80 | (g & 0x3f);
                }
                return 3;
        } else if (g < (1 << 21)) {
                if (out_utf8) {
                        out_utf8[0] = 0xf0 | ((g >> 18) & 0x07);
                        out_utf8[1] = 0x80 | ((g >> 12) & 0x3f);
                        out_utf8[2] = 0x80 | ((g >> 6) & 0x3f);
                        out_utf8[3] = 0x80 | (g & 0x3f);
                }
                return 4;
        }

        return 0;
}

char* utf16_to_utf8(const char16_t *s, size_t length /* bytes! */) {
        const uint8_t *f;
        char *r, *t;

        if (length == 0)
                return new0(char, 1);

        assert(s);

        if (length == SIZE_MAX) {
                length = char16_strlen(s);

                if (length > SIZE_MAX/2)
                        return NULL; /* overflow */

                length *= 2;
        }

        /* Input length is in bytes, i.e. the shortest possible character takes 2 bytes. Each unicode character may
         * take up to 4 bytes in UTF-8. Let's also account for a trailing NUL byte. */
        if (length > (SIZE_MAX - 1) / 2)
                return NULL; /* overflow */

        r = new(char, length * 2 + 1);
        if (!r)
                return NULL;

        f = (const uint8_t*) s;
        t = r;

        while (f + 1 < (const uint8_t*) s + length) {
                char16_t w1, w2;

                /* see RFC 2781 section 2.2 */

                w1 = f[1] << 8 | f[0];
                f += 2;

                if (!utf16_is_surrogate(w1)) {
                        t += utf8_encode_unichar(t, w1);
                        continue;
                }

                if (utf16_is_trailing_surrogate(w1))
                        continue; /* spurious trailing surrogate, ignore */

                if (f + 1 >= (const uint8_t*) s + length)
                        break;

                w2 = f[1] << 8 | f[0];
                f += 2;

                if (!utf16_is_trailing_surrogate(w2)) {
                        f -= 2;
                        continue; /* surrogate missing its trailing surrogate, ignore */
                }

                t += utf8_encode_unichar(t, utf16_surrogate_pair_to_unichar(w1, w2));
        }

        *t = 0;
        return r;
}

size_t utf16_encode_unichar(char16_t *out, char32_t c) {

        /* Note that this encodes as little-endian. */

        switch (c) {

        case 0 ... 0xd7ffU:
        case 0xe000U ... 0xffffU:
                out[0] = htole16(c);
                return 1;

        case 0x10000U ... 0x10ffffU:
                c -= 0x10000U;
                out[0] = htole16((c >> 10) + 0xd800U);
                out[1] = htole16((c & 0x3ffU) + 0xdc00U);
                return 2;

        default: /* A surrogate (invalid) */
                return 0;
        }
}

char16_t *utf8_to_utf16(const char *s, size_t length) {
        char16_t *n, *p;
        int r;

        if (length == 0)
                return new0(char16_t, 1);

        assert(s);

        if (length == SIZE_MAX)
                length = strlen(s);

        if (length > SIZE_MAX - 1)
                return NULL; /* overflow */

        n = new(char16_t, length + 1);
        if (!n)
                return NULL;

        p = n;

        for (size_t i = 0; i < length;) {
                char32_t unichar;
                size_t e;

                e = utf8_encoded_expected_len(s[i]);
                if (e <= 1) /* Invalid and single byte characters are copied as they are */
                        goto copy;

                if (i + e > length) /* sequence longer than input buffer, then copy as-is */
                        goto copy;

                r = utf8_encoded_to_unichar(s + i, &unichar);
                if (r < 0) /* sequence invalid, then copy as-is */
                        goto copy;

                p += utf16_encode_unichar(p, unichar);
                i += e;
                continue;

        copy:
                *(p++) = htole16(s[i++]);
        }

        *p = 0;
        return n;
}

size_t char16_strlen(const char16_t *s) {
        size_t n = 0;

        assert(s);

        while (*s != 0)
                n++, s++;

        return n;
}

size_t char16_strsize(const char16_t *s) {
        return s ? (char16_strlen(s) + 1) * sizeof(*s) : 0;
}

/* expected size used to encode one unicode char */
static int utf8_unichar_to_encoded_len(char32_t unichar) {

        if (unichar < 0x80)
                return 1;
        if (unichar < 0x800)
                return 2;
        if (unichar < 0x10000)
                return 3;
        if (unichar < 0x200000)
                return 4;
        if (unichar < 0x4000000)
                return 5;

        return 6;
}

/* validate one encoded unicode char and return its length */
int utf8_encoded_valid_unichar(const char *str, size_t length /* bytes */) {
        char32_t unichar;
        size_t len;
        int r;

        assert(str);
        assert(length > 0);

        /* We read until NUL, at most length bytes. SIZE_MAX may be used to disable the length check. */

        len = utf8_encoded_expected_len(str[0]);
        if (len == 0)
                return -EINVAL;

        /* Do we have a truncated multi-byte character? */
        if (len > length)
                return -EINVAL;

        /* ascii is valid */
        if (len == 1)
                return 1;

        /* check if expected encoded chars are available */
        for (size_t i = 0; i < len; i++)
                if ((str[i] & 0x80) != 0x80)
                        return -EINVAL;

        r = utf8_encoded_to_unichar(str, &unichar);
        if (r < 0)
                return r;

        /* check if encoded length matches encoded value */
        if (utf8_unichar_to_encoded_len(unichar) != (int) len)
                return -EINVAL;

        /* check if value has valid range */
        if (!unichar_is_valid(unichar))
                return -EINVAL;

        return (int) len;
}

size_t utf8_n_codepoints(const char *str) {
        size_t n = 0;

        /* Returns the number of UTF-8 codepoints in this string, or SIZE_MAX if the string is not valid UTF-8. */

        while (*str != 0) {
                int k;

                k = utf8_encoded_valid_unichar(str, SIZE_MAX);
                if (k < 0)
                        return SIZE_MAX;

                str += k;
                n++;
        }

        return n;
}

size_t utf8_console_width(const char *str) {

        if (isempty(str))
                return 0;

        /* Returns the approximate width a string will take on screen when printed on a character cell
         * terminal/console. */

        size_t n = 0;
        while (*str) {
                int w;

                w = utf8_char_console_width(str);
                if (w < 0)
                        return SIZE_MAX;

                n += w;
                str = utf8_next_char(str);
        }

        return n;
}

size_t utf8_last_length(const char *s, size_t n) {
        int r;

        if (n == SIZE_MAX)
                n = strlen(s);

        /* Determines length in bytes of last UTF-8 codepoint in string. If the string is empty, returns
         * zero. Treats invalid UTF-8 codepoints as 1 sized ones. */

        for (size_t last = 0;;) {
                if (n == 0)
                        return last;

                r = utf8_encoded_valid_unichar(s, n);
                if (r <= 0)
                        r = 1; /* treat invalid UTF-8 as byte-wide */

                s += r;
                n -= r;
                last = r;
        }
}

1	/* SPDX-License-Identifier: LGPL-2.0-or-later */
2
3	/* Parts of this file are based on the GLIB utf8 validation functions. The original copyright follows.
4	*
5	* gutf8.c - Operations on UTF-8 strings.
6	* Copyright (C) 1999 Tom Tromey
7	* Copyright (C) 2000 Red Hat, Inc.
8	*/
9
10	#include <errno.h>
11	#include <stdbool.h>
12	#include <stdlib.h>
13
14	#include "alloc-util.h"
15	#include "gunicode.h"
16	#include "hexdecoct.h"
17	#include "macro.h"
18	#include "string-util.h"
19	#include "utf8.h"
20
21	bool unichar_is_valid(char32_t ch) {	71,928✔
22
23	if (ch >= 0x110000) /* End of unicode space */	71,928✔
24	return false;
25	if ((ch & 0xFFFFF800) == 0xD800) /* Reserved area for UTF-16 */	71,928✔
26	return false;
27	if ((ch >= 0xFDD0) && (ch <= 0xFDEF)) /* Reserved */	71,928✔
28	return false;
29	if ((ch & 0xFFFE) == 0xFFFE) /* BOM (Byte Order Mark) */	71,928✔
30	return false;	6✔
31
32	return true;
33	}
34
35	static bool unichar_is_control(char32_t ch) {	95,395,432✔
36
37	/*
38	0 to ' '-1 is the C0 range.
39	DEL=0x7F, and DEL+1 to 0x9F is C1 range.
40	'\t' is in C0 range, but more or less harmless and commonly used.
41	*/
42
43	return (ch < ' ' && !IN_SET(ch, '\t', '\n')) \|\|	95,395,432✔
44	(0x7F <= ch && ch <= 0x9F);	95,395,260✔
45	}
46
47	/* count of characters used to encode one unicode char */
48	static size_t utf8_encoded_expected_len(uint8_t c) {	623,663,809✔
49	if (c < 0x80)	623,663,809✔
50	return 1;
51	if ((c & 0xe0) == 0xc0)	235,245✔
52	return 2;
53	if ((c & 0xf0) == 0xe0)	209,006✔
54	return 3;
55	if ((c & 0xf8) == 0xf0)	993✔
56	return 4;
57	if ((c & 0xfc) == 0xf8)	64✔
58	return 5;
59	if ((c & 0xfe) == 0xfc)	64✔
60	return 6;	×
61
62	return 0;
63	}
64
65	/* decode one unicode char */
66	int utf8_encoded_to_unichar(const char str, char32_t ret_unichar) {	100,749,776✔
67	char32_t unichar;	100,749,776✔
68	size_t len;	100,749,776✔
69
70	assert(str);	100,749,776✔
71
72	len = utf8_encoded_expected_len(str[0]);	100,749,776✔
73
74	switch (len) {	100,749,776✔
75	case 1:	100,587,718✔
76	*ret_unichar = (char32_t)str[0];	100,587,718✔
77	return 1;	100,587,718✔
78	case 2:	21,814✔
79	unichar = str[0] & 0x1f;	21,814✔
80	break;	21,814✔
81	case 3:	139,689✔
82	unichar = (char32_t)str[0] & 0x0f;	139,689✔
83	break;	139,689✔
84	case 4:	555✔
85	unichar = (char32_t)str[0] & 0x07;	555✔
86	break;	555✔
87	case 5:	×
88	unichar = (char32_t)str[0] & 0x03;	×
89	break;	×
90	case 6:	×
91	unichar = (char32_t)str[0] & 0x01;	×
92	break;	×
93	default:
94	return -EINVAL;
95	}
96
97	for (size_t i = 1; i < len; i++) {	464,907✔
98	if (((char32_t)str[i] & 0xc0) != 0x80)	302,855✔
99	return -EINVAL;
100
101	unichar <<= 6;	302,849✔
102	unichar \|= (char32_t)str[i] & 0x3f;	302,849✔
103	}
104
105	*ret_unichar = unichar;	162,052✔
106	return len;	162,052✔
107	}
108
109	bool utf8_is_printable_newline(const char* str, size_t length, bool allow_newline) {	3,153,048✔
110	assert(str);	3,153,048✔
111
112	for (const char *p = str; length > 0;) {	98,548,308✔
113	int encoded_len;	95,395,433✔
114	char32_t val;	95,395,433✔
115
116	encoded_len = utf8_encoded_valid_unichar(p, length);	95,395,433✔
117	if (encoded_len < 0)	95,395,433✔
118	return false;	173✔
119	assert(encoded_len > 0 && (size_t) encoded_len <= length);	95,395,432✔
120
121	if (utf8_encoded_to_unichar(p, &val) < 0 \|\|	95,395,432✔
122	unichar_is_control(val) \|\|	95,395,432✔
123	(!allow_newline && val == '\n'))	21,196,410✔
124	return false;
125
126	length -= encoded_len;	95,395,260✔
127	p += encoded_len;	95,395,260✔
128	}
129
130	return true;
131	}
132
133	char* utf8_is_valid_n(const char *str, size_t len_bytes) {	24,876,772✔
134	/* Check if the string is composed of valid utf8 characters. If length len_bytes is given, stop after
135	* len_bytes. Otherwise, stop at NUL. */
136
137	assert(str);	24,876,772✔
138
139	for (size_t i = 0; len_bytes != SIZE_MAX ? i < len_bytes : str[i] != '\0'; ) {	431,236,824✔
140	int len;	406,361,181✔
141
142	if (_unlikely_(str[i] == '\0'))	406,361,181✔
143	return NULL; /* embedded NUL */
144
145	len = utf8_encoded_valid_unichar(str + i,	406,361,178✔
146	len_bytes != SIZE_MAX ? len_bytes - i : SIZE_MAX);
147	if (_unlikely_(len < 0))	406,361,178✔
148	return NULL; /* invalid character */
149
150	i += len;	406,360,052✔
151	}
152
153	return (char*) str;
154	}
155
156	char* utf8_escape_invalid(const char *str) {	3,178✔
157	char p, s;	3,178✔
158
159	assert(str);	3,178✔
160
161	p = s = malloc(strlen(str) * 4 + 1);	3,178✔
162	if (!p)	3,178✔
163	return NULL;
164
165	while (*str) {	72,345✔
166	int len;	69,167✔
167
168	len = utf8_encoded_valid_unichar(str, SIZE_MAX);	69,167✔
169	if (len > 0) {	69,167✔
170	s = mempcpy(s, str, len);	69,151✔
171	str += len;	69,151✔
172	} else {
173	s = stpcpy(s, UTF8_REPLACEMENT_CHARACTER);	16✔
174	str += 1;	16✔
175	}
176	}
177
178	*s = '\0';	3,178✔
179	return str_realloc(p);	3,178✔
180	}
181
182	int utf8_char_console_width(const char *str) {	4,038,506✔
183	char32_t c;	4,038,506✔
184	int r;	4,038,506✔
185
186	r = utf8_encoded_to_unichar(str, &c);	4,038,506✔
187	if (r < 0)	4,038,506✔
188	return r;	4,038,506✔
189
190	if (c == '\t')	4,038,505✔
191	return 8; /* Assume a tab width of 8 */
192
193	/* TODO: we should detect combining characters */
194
195	return unichar_iswide(c) ? 2 : 1;	4,030,133✔
196	}
197
198	char* utf8_escape_non_printable_full(const char *str, size_t console_width, bool force_ellipsis) {	522✔
199	char p, s, *prev_s;	522✔
200	size_t n = 0; /* estimated print width */	522✔
201
202	assert(str);	522✔
203
204	if (console_width == 0)	522✔
205	return strdup("");	9✔
206
207	p = s = prev_s = malloc(strlen(str) * 4 + 1);	513✔
208	if (!p)	513✔
209	return NULL;
210
211	for (;;) {	19,240✔
212	int len;	19,240✔
213	char *saved_s = s;	19,240✔
214
215	if (!str) { / done! */	19,240✔
216	if (force_ellipsis)	439✔
217	goto truncation;	73✔
218	else
219	goto finish;	366✔
220	}
221
222	len = utf8_encoded_valid_unichar(str, SIZE_MAX);	18,801✔
223	if (len > 0) {	18,801✔
224	if (utf8_is_printable(str, len)) {	18,705✔
225	int w;	18,542✔
226
227	w = utf8_char_console_width(str);	18,542✔
228	assert(w >= 0);	18,542✔
229	if (n + w > console_width)	18,542✔
230	goto truncation;	34✔
231
232	s = mempcpy(s, str, len);	18,508✔
233	str += len;	18,508✔
234	n += w;	18,508✔
235
236	} else {
237	for (; len > 0; len--) {	288✔
238	if (n + 4 > console_width)	163✔
239	goto truncation;	38✔
240
241	*(s++) = '\\';	125✔
242	*(s++) = 'x';	125✔
243	(s++) = hexchar((int) str >> 4);	125✔
244	(s++) = hexchar((int) str);	125✔
245
246	str += 1;	125✔
247	n += 4;	125✔
248	}
249	}
250	} else {
251	if (n + 1 > console_width)	96✔
252	goto truncation;	2✔
253
254	s = mempcpy(s, UTF8_REPLACEMENT_CHARACTER, strlen(UTF8_REPLACEMENT_CHARACTER));	94✔
255	str += 1;	94✔
256	n += 1;	94✔
257	}
258
259	prev_s = saved_s;
260	}
261
262	truncation:	147✔
263	/* Try to go back one if we don't have enough space for the ellipsis */
264	if (n + 1 > console_width)	147✔
265	s = prev_s;	88✔
266
267	s = mempcpy(s, "…", strlen("…"));	147✔
268
269	finish:	513✔
270	*s = '\0';	513✔
271	return str_realloc(p);	513✔
272	}
273
274	char* ascii_is_valid_n(const char *str, size_t len) {	58,903✔
275	/* Check whether the string consists of valid ASCII bytes, i.e values between 1 and 127, inclusive.
276	* Stops at len, or NUL byte if len is SIZE_MAX. */
277
278	assert(str);	58,903✔
279
280	for (size_t i = 0; len != SIZE_MAX ? i < len : str[i] != '\0'; i++)	1,499,226✔
281	if ((unsigned char) str[i] >= 128 \|\| str[i] == '\0')	1,442,958✔
282	return NULL;
283
284	return (char*) str;
285	}
286
287	int utf8_to_ascii(const char str, char replacement_char, char *ret) {	28✔
288	/* Convert to a string that has only ASCII chars, replacing anything that is not ASCII
289	* by replacement_char. */
290
291	_cleanup_free_ char *ans = new(char, strlen(str) + 1);	56✔
292	if (!ans)	28✔
293	return -ENOMEM;
294
295	char *q = ans;
296
297	for (const char p = str; p; q++) {	182✔
298	int l;	156✔
299
300	l = utf8_encoded_valid_unichar(p, SIZE_MAX);	156✔
301	if (l < 0) /* Non-UTF-8, let's not even try to propagate the garbage */	156✔
302	return l;
303
304	if (l == 1)	154✔
305	q = p;	137✔
306	else
307	/* non-ASCII, we need to replace it */
308	*q = replacement_char;	17✔
309
310	p += l;	154✔
311	}
312	*q = '\0';	26✔
313
314	*ret = TAKE_PTR(ans);	26✔
315	return 0;	26✔
316	}
317
318	/**
319	* utf8_encode_unichar() - Encode single UCS-4 character as UTF-8
320	* @out_utf8: output buffer of at least 4 bytes or NULL
321	* @g: UCS-4 character to encode
322	*
323	* This encodes a single UCS-4 character as UTF-8 and writes it into @out_utf8.
324	* The length of the character is returned. It is not zero-terminated! If the
325	* output buffer is NULL, only the length is returned.
326	*
327	* Returns: The length in bytes that the UTF-8 representation does or would
328	* occupy.
329	*/
330	size_t utf8_encode_unichar(char *out_utf8, char32_t g) {	68,342✔
331
332	if (g < (1 << 7)) {	68,342✔
333	if (out_utf8)	68,218✔
334	out_utf8[0] = g & 0x7f;	68,218✔
335	return 1;	68,218✔
336	} else if (g < (1 << 11)) {	124✔
337	if (out_utf8) {	17✔
338	out_utf8[0] = 0xc0 \| ((g >> 6) & 0x1f);	17✔
339	out_utf8[1] = 0x80 \| (g & 0x3f);	17✔
340	}
341	return 2;	17✔
342	} else if (g < (1 << 16)) {	107✔
343	if (out_utf8) {	96✔
344	out_utf8[0] = 0xe0 \| ((g >> 12) & 0x0f);	96✔
345	out_utf8[1] = 0x80 \| ((g >> 6) & 0x3f);	96✔
346	out_utf8[2] = 0x80 \| (g & 0x3f);	96✔
347	}
348	return 3;	96✔
349	} else if (g < (1 << 21)) {	11✔
350	if (out_utf8) {	11✔
351	out_utf8[0] = 0xf0 \| ((g >> 18) & 0x07);	11✔
352	out_utf8[1] = 0x80 \| ((g >> 12) & 0x3f);	11✔
353	out_utf8[2] = 0x80 \| ((g >> 6) & 0x3f);	11✔
354	out_utf8[3] = 0x80 \| (g & 0x3f);	11✔
355	}
356	return 4;	11✔
357	}
358
359	return 0;
360	}
361
362	char* utf16_to_utf8(const char16_t s, size_t length / bytes! */) {	1,931✔
363	const uint8_t *f;	1,931✔
364	char r, t;	1,931✔
365
366	if (length == 0)	1,931✔
367	return new0(char, 1);	×
368
369	assert(s);	1,931✔
370
371	if (length == SIZE_MAX) {	1,931✔
372	length = char16_strlen(s);	6✔
373
374	if (length > SIZE_MAX/2)	6✔
375	return NULL; /* overflow */
376
377	length *= 2;	6✔
378	}
379
380	/* Input length is in bytes, i.e. the shortest possible character takes 2 bytes. Each unicode character may
381	* take up to 4 bytes in UTF-8. Let's also account for a trailing NUL byte. */
382	if (length > (SIZE_MAX - 1) / 2)	1,931✔
383	return NULL; /* overflow */
384
385	r = new(char, length * 2 + 1);	1,931✔
386	if (!r)	1,931✔
387	return NULL;
388
389	f = (const uint8_t*) s;
390	t = r;
391
392	while (f + 1 < (const uint8_t*) s + length) {	70,084✔
393	char16_t w1, w2;	68,153✔
394
395	/* see RFC 2781 section 2.2 */
396
397	w1 = f[1] << 8 \| f[0];	68,153✔
398	f += 2;	68,153✔
399
400	if (!utf16_is_surrogate(w1)) {	68,153✔
401	t += utf8_encode_unichar(t, w1);	68,146✔
402	continue;	68,146✔
403	}
404
405	if (utf16_is_trailing_surrogate(w1))	7✔
406	continue; /* spurious trailing surrogate, ignore */	1✔
407
408	if (f + 1 >= (const uint8_t*) s + length)	6✔
409	break;
410
411	w2 = f[1] << 8 \| f[0];	6✔
412	f += 2;	6✔
413
414	if (!utf16_is_trailing_surrogate(w2)) {	6✔
415	f -= 2;	1✔
416	continue; /* surrogate missing its trailing surrogate, ignore */	1✔
417	}
418
419	t += utf8_encode_unichar(t, utf16_surrogate_pair_to_unichar(w1, w2));	5✔
420	}
421
422	*t = 0;	1,931✔
423	return r;	1,931✔
424	}
425
426	size_t utf16_encode_unichar(char16_t *out, char32_t c) {	16✔
427
428	/* Note that this encodes as little-endian. */
429
430	switch (c) {	16✔
431
432	case 0 ... 0xd7ffU:	12✔
433	case 0xe000U ... 0xffffU:
434	out[0] = htole16(c);	12✔
435	return 1;	12✔
436
437	case 0x10000U ... 0x10ffffU:	4✔
438	c -= 0x10000U;	4✔
439	out[0] = htole16((c >> 10) + 0xd800U);	4✔
440	out[1] = htole16((c & 0x3ffU) + 0xdc00U);	4✔
441	return 2;	4✔
442
443	default: /* A surrogate (invalid) */
444	return 0;
445	}
446	}
447
448	char16_t utf8_to_utf16(const char s, size_t length) {	108✔
449	char16_t n, p;	108✔
450	int r;	108✔
451
452	if (length == 0)	108✔
453	return new0(char16_t, 1);	×
454
455	assert(s);	108✔
456
457	if (length == SIZE_MAX)	108✔
458	length = strlen(s);	107✔
459
460	if (length > SIZE_MAX - 1)	107✔
461	return NULL; /* overflow */
462
463	n = new(char16_t, length + 1);	108✔
464	if (!n)	108✔
465	return NULL;
466
467	p = n;
468
469	for (size_t i = 0; i < length;) {	1,144✔
470	char32_t unichar;	1,036✔
471	size_t e;	1,036✔
472
473	e = utf8_encoded_expected_len(s[i]);	1,036✔
474	if (e <= 1) /* Invalid and single byte characters are copied as they are */	1,036✔
475	goto copy;	1,020✔
476
477	if (i + e > length) /* sequence longer than input buffer, then copy as-is */	16✔
478	goto copy;	×
479
480	r = utf8_encoded_to_unichar(s + i, &unichar);	16✔
481	if (r < 0) /* sequence invalid, then copy as-is */	16✔
482	goto copy;	×
483
484	p += utf16_encode_unichar(p, unichar);	16✔
485	i += e;	16✔
486	continue;	16✔
487
488	copy:	1,020✔
489	*(p++) = htole16(s[i++]);	1,020✔
490	}
491
492	*p = 0;	108✔
493	return n;	108✔
494	}
495
496	size_t char16_strlen(const char16_t *s) {	334✔
497	size_t n = 0;	334✔
498
499	assert(s);	334✔
500
501	while (*s != 0)	4,107✔
502	n++, s++;	3,773✔
503
504	return n;	334✔
505	}
506
507	size_t char16_strsize(const char16_t *s) {	3✔
508	return s ? (char16_strlen(s) + 1) * sizeof(*s) : 0;	3✔
509	}
510
511	/* expected size used to encode one unicode char */
512	static int utf8_unichar_to_encoded_len(char32_t unichar) {	71,918✔
513
514	if (unichar < 0x80)	71,918✔
515	return 1;
516	if (unichar < 0x800)	71,918✔
517	return 2;
518	if (unichar < 0x10000)	67,820✔
519	return 3;
520	if (unichar < 0x200000)	368✔
521	return 4;
522	if (unichar < 0x4000000)	×
523	return 5;	×
524
525	return 6;
526	}
527
528	/* validate one encoded unicode char and return its length */
529	int utf8_encoded_valid_unichar(const char str, size_t length / bytes */) {	522,912,997✔
530	char32_t unichar;	522,912,997✔
531	size_t len;	522,912,997✔
532	int r;	522,912,997✔
533
534	assert(str);	522,912,997✔
535	assert(length > 0);	522,912,997✔
536
537	/* We read until NUL, at most length bytes. SIZE_MAX may be used to disable the length check. */
538
539	len = utf8_encoded_expected_len(str[0]);	522,912,997✔
540	if (len == 0)	522,912,997✔
541	return -EINVAL;	522,912,997✔
542
543	/* Do we have a truncated multi-byte character? */
544	if (len > length)	522,912,933✔
545	return -EINVAL;
546
547	/* ascii is valid */
548	if (len == 1)	522,912,925✔
549	return 1;
550
551	/* check if expected encoded chars are available */
552	for (size_t i = 0; i < len; i++)	286,768✔
553	if ((str[i] & 0x80) != 0x80)	214,845✔
554	return -EINVAL;
555
556	r = utf8_encoded_to_unichar(str, &unichar);	71,923✔
557	if (r < 0)	71,923✔
558	return r;
559
560	/* check if encoded length matches encoded value */
561	if (utf8_unichar_to_encoded_len(unichar) != (int) len)	71,918✔
562	return -EINVAL;
563
564	/* check if value has valid range */
565	if (!unichar_is_valid(unichar))	71,918✔
566	return -EINVAL;	6✔
567
568	return (int) len;
569	}
570
571	size_t utf8_n_codepoints(const char *str) {	6✔
572	size_t n = 0;	6✔
573
574	/* Returns the number of UTF-8 codepoints in this string, or SIZE_MAX if the string is not valid UTF-8. */
575
576	while (*str != 0) {	34✔
577	int k;	29✔
578
579	k = utf8_encoded_valid_unichar(str, SIZE_MAX);	29✔
580	if (k < 0)	29✔
581	return SIZE_MAX;
582
583	str += k;	28✔
584	n++;	28✔
585	}
586
587	return n;
588	}
589
590	size_t utf8_console_width(const char *str) {	273,496✔
591
592	if (isempty(str))	273,496✔
593	return 0;
594
595	/* Returns the approximate width a string will take on screen when printed on a character cell
596	* terminal/console. */
597
598	size_t n = 0;
599	while (*str) {	3,787,880✔
600	int w;	3,526,050✔
601
602	w = utf8_char_console_width(str);	3,526,050✔
603	if (w < 0)	3,526,050✔
604	return SIZE_MAX;
605
606	n += w;	3,526,049✔
607	str = utf8_next_char(str);	3,526,049✔
608	}
609
610	return n;
611	}
612
613	size_t utf8_last_length(const char *s, size_t n) {	9✔
614	int r;	9✔
615
616	if (n == SIZE_MAX)	9✔
617	n = strlen(s);	7✔
618
619	/* Determines length in bytes of last UTF-8 codepoint in string. If the string is empty, returns
620	* zero. Treats invalid UTF-8 codepoints as 1 sized ones. */
621
622	for (size_t last = 0;;) {	19✔
623	if (n == 0)	28✔
624	return last;	9✔
625
626	r = utf8_encoded_valid_unichar(s, n);	19✔
627	if (r <= 0)	19✔
628	r = 1; /* treat invalid UTF-8 as byte-wide */	×
629
630	s += r;	19✔
631	n -= r;	19✔
632	last = r;	19✔
633	}
634	}

systemd / systemd / 13380515387

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous