23825567702

Committed 31 Mar 2026 12:42PM UTC coverage: 72.404% (+0.006%) from 72.398%

Build # 23825567702

Build Type

push

github

Committed by

daandemeyer

Commit Message

terminal-util: fix boot hang from ANSI terminal size queries

Since v257, terminal_fix_size() is called during early boot via
console_setup() → reset_dev_console_fd() to query terminal dimensions
via ANSI escape sequences. This has caused intermittent boot hangs
where the system gets stuck with a blinking cursor and requires a
keypress to continue (see systemd/systemd#35499).

The function tries CSI 18 first, then falls back to DSR if that fails.
Previously, each method independently opened a non-blocking fd, disabled
echo/icanon, ran its query, restored termios, and closed its fd. This
created two problems:

1. Echo window between CSI 18 and DSR fallback: After CSI 18 times out
   and restores termios (re-enabling ECHO and ICANON), there is a brief
   window before DSR disables them again. If the terminal's CSI 18
   response arrives during this window, it is echoed back to the
   terminal — where the terminal interprets \e[8;rows;cols t as a
   "resize text area" command — and the response bytes land in the
   canonical line buffer as stale input that can confuse the DSR
   response parser.

2. Cursor left at bottom-right on DSR timeout: The DSR method worked by
   sending two DSR queries — one to save the cursor position, then
   moving the cursor to (32766,32766) and sending another to read the
   clamped position. If neither response was received (timeout), the
   cursor restore was skipped (conditional on saved_row > 0), leaving
   the cursor at the bottom-right corner of the terminal. The
   subsequent terminal_reset_ansi_seq() then moved it to the beginning
   of the last line via \e[1G, making boot output appear at the bottom
   of the screen — giving the appearance of a hang even when the system
   was still booting.

This commit fixes both issues:

- terminal_fix_size() now opens the non-blocking fd and configures
  termios once for both query methods, so echo stays disabled for the
  entire CSI 18 → DSR fallback sequence with no gap. tcflu... (continued)

Coverage Stats

22 of 57 new or added lines in 3 files covered. (38.6%)

834 existing lines in 52 files now uncovered.

318485 of 439872 relevant lines covered (72.4%)

1162379.76 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.8

/src/basic/compress.c

/* SPDX-License-Identifier: LGPL-2.1-or-later */

#include <stdio.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>

#if HAVE_LZ4
#include <lz4.h>
#include <lz4hc.h>
#include <lz4frame.h>
#endif

#if HAVE_XZ
#include <lzma.h>
#endif

#if HAVE_ZSTD
#include <zstd.h>
#include <zstd_errors.h>
#endif

#include "sd-dlopen.h"

#include "alloc-util.h"
#include "bitfield.h"
#include "compress.h"
#include "dlfcn-util.h"
#include "fileio.h"
#include "io-util.h"
#include "log.h"
#include "string-table.h"
#include "string-util.h"
#include "unaligned.h"

#if HAVE_LZ4
static void *lz4_dl = NULL;

static DLSYM_PROTOTYPE(LZ4F_compressBegin) = NULL;
static DLSYM_PROTOTYPE(LZ4F_compressBound) = NULL;
static DLSYM_PROTOTYPE(LZ4F_compressEnd) = NULL;
static DLSYM_PROTOTYPE(LZ4F_compressUpdate) = NULL;
static DLSYM_PROTOTYPE(LZ4F_createCompressionContext) = NULL;
static DLSYM_PROTOTYPE(LZ4F_createDecompressionContext) = NULL;
static DLSYM_PROTOTYPE(LZ4F_decompress) = NULL;
static DLSYM_PROTOTYPE(LZ4F_freeCompressionContext) = NULL;
static DLSYM_PROTOTYPE(LZ4F_freeDecompressionContext) = NULL;
static DLSYM_PROTOTYPE(LZ4F_isError) = NULL;
static DLSYM_PROTOTYPE(LZ4_compress_HC) = NULL;
/* These are used in test-compress.c so we don't make them static. */
// NOLINTBEGIN(misc-use-internal-linkage)
DLSYM_PROTOTYPE(LZ4_compress_default) = NULL;
DLSYM_PROTOTYPE(LZ4_decompress_safe) = NULL;
DLSYM_PROTOTYPE(LZ4_decompress_safe_partial) = NULL;
DLSYM_PROTOTYPE(LZ4_versionNumber) = NULL;
// NOLINTEND(misc-use-internal-linkage)

DEFINE_TRIVIAL_CLEANUP_FUNC_FULL_RENAME(LZ4F_compressionContext_t, sym_LZ4F_freeCompressionContext, LZ4F_freeCompressionContextp, NULL);
DEFINE_TRIVIAL_CLEANUP_FUNC_FULL_RENAME(LZ4F_decompressionContext_t, sym_LZ4F_freeDecompressionContext, LZ4F_freeDecompressionContextp, NULL);
#endif

#if HAVE_ZSTD
static void *zstd_dl = NULL;

static DLSYM_PROTOTYPE(ZSTD_CCtx_setParameter) = NULL;
static DLSYM_PROTOTYPE(ZSTD_compress) = NULL;
static DLSYM_PROTOTYPE(ZSTD_compressStream2) = NULL;
static DLSYM_PROTOTYPE(ZSTD_createCCtx) = NULL;
static DLSYM_PROTOTYPE(ZSTD_createDCtx) = NULL;
static DLSYM_PROTOTYPE(ZSTD_CStreamInSize) = NULL;
static DLSYM_PROTOTYPE(ZSTD_CStreamOutSize) = NULL;
static DLSYM_PROTOTYPE(ZSTD_decompressStream) = NULL;
static DLSYM_PROTOTYPE(ZSTD_DStreamInSize) = NULL;
static DLSYM_PROTOTYPE(ZSTD_DStreamOutSize) = NULL;
static DLSYM_PROTOTYPE(ZSTD_freeCCtx) = NULL;
static DLSYM_PROTOTYPE(ZSTD_freeDCtx) = NULL;
static DLSYM_PROTOTYPE(ZSTD_getErrorCode) = NULL;
static DLSYM_PROTOTYPE(ZSTD_getErrorName) = NULL;
static DLSYM_PROTOTYPE(ZSTD_getFrameContentSize) = NULL;
static DLSYM_PROTOTYPE(ZSTD_isError) = NULL;

DEFINE_TRIVIAL_CLEANUP_FUNC_FULL_RENAME(ZSTD_CCtx*, sym_ZSTD_freeCCtx, ZSTD_freeCCtxp, NULL);
DEFINE_TRIVIAL_CLEANUP_FUNC_FULL_RENAME(ZSTD_DCtx*, sym_ZSTD_freeDCtx, ZSTD_freeDCtxp, NULL);

static int zstd_ret_to_errno(size_t ret) {
        switch (sym_ZSTD_getErrorCode(ret)) {
        case ZSTD_error_dstSize_tooSmall:
                return -ENOBUFS;
        case ZSTD_error_memory_allocation:
                return -ENOMEM;
        default:
                return -EBADMSG;
        }
}
#endif

#if HAVE_XZ
static void *lzma_dl = NULL;

static DLSYM_PROTOTYPE(lzma_code) = NULL;
static DLSYM_PROTOTYPE(lzma_easy_encoder) = NULL;
static DLSYM_PROTOTYPE(lzma_end) = NULL;
static DLSYM_PROTOTYPE(lzma_stream_buffer_encode) = NULL;
static DLSYM_PROTOTYPE(lzma_stream_decoder) = NULL;
static DLSYM_PROTOTYPE(lzma_lzma_preset) = NULL;

/* We can't just do _cleanup_(sym_lzma_end) because a compiler bug makes
 * this fail with:
 * ../src/basic/compress.c: In function ‘decompress_blob_xz’:
 * ../src/basic/compress.c:304:9: error: cleanup argument not a function
 *   304 |         _cleanup_(sym_lzma_end) lzma_stream s = LZMA_STREAM_INIT;
 *       |         ^~~~~~~~~
 */
static inline void lzma_end_wrapper(lzma_stream *ls) {
        sym_lzma_end(ls);
}
#endif

#define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))

static const char* const compression_table[_COMPRESSION_MAX] = {
        [COMPRESSION_NONE] = "NONE",
        [COMPRESSION_XZ]   = "XZ",
        [COMPRESSION_LZ4]  = "LZ4",
        [COMPRESSION_ZSTD] = "ZSTD",
};

static const char* const compression_lowercase_table[_COMPRESSION_MAX] = {
        [COMPRESSION_NONE] = "none",
        [COMPRESSION_XZ]   = "xz",
        [COMPRESSION_LZ4]  = "lz4",
        [COMPRESSION_ZSTD] = "zstd",
};

DEFINE_STRING_TABLE_LOOKUP(compression, Compression);
DEFINE_STRING_TABLE_LOOKUP(compression_lowercase, Compression);

bool compression_supported(Compression c) {
        static const unsigned supported =
                (1U << COMPRESSION_NONE) |
                (1U << COMPRESSION_XZ) * HAVE_XZ |
                (1U << COMPRESSION_LZ4) * HAVE_LZ4 |
                (1U << COMPRESSION_ZSTD) * HAVE_ZSTD;

        assert(c >= 0);
        assert(c < _COMPRESSION_MAX);

        return BIT_SET(supported, c);
}

int dlopen_lzma(void) {
#if HAVE_XZ
        SD_ELF_NOTE_DLOPEN(
                        "lzma",
                        "Support lzma compression in journal and coredump files",
                        COMPRESSION_PRIORITY_XZ,
                        "liblzma.so.5");

        return dlopen_many_sym_or_warn(
                        &lzma_dl,
                        "liblzma.so.5", LOG_DEBUG,
                        DLSYM_ARG(lzma_code),
                        DLSYM_ARG(lzma_easy_encoder),
                        DLSYM_ARG(lzma_end),
                        DLSYM_ARG(lzma_stream_buffer_encode),
                        DLSYM_ARG(lzma_lzma_preset),
                        DLSYM_ARG(lzma_stream_decoder));
#else
        return -EOPNOTSUPP;
#endif
}

int compress_blob_xz(const void *src, uint64_t src_size,
                     void *dst, size_t dst_alloc_size, size_t *dst_size, int level) {

        assert(src);
        assert(src_size > 0);
        assert(dst);
        assert(dst_alloc_size > 0);
        assert(dst_size);

#if HAVE_XZ
        lzma_options_lzma opt = {
                1u << 20u, NULL, 0, LZMA_LC_DEFAULT, LZMA_LP_DEFAULT,
                LZMA_PB_DEFAULT, LZMA_MODE_FAST, 128, LZMA_MF_HC3, 4
        };
        lzma_filter filters[] = {
                { LZMA_FILTER_LZMA2, &opt },
                { LZMA_VLI_UNKNOWN, NULL }
        };
        lzma_ret ret;
        size_t out_pos = 0;
        int r;

        r = dlopen_lzma();
        if (r < 0)
                return r;

        if (level >= 0) {
                r = sym_lzma_lzma_preset(&opt, (uint32_t) level);
                if (r < 0)
                        return r;
        }

        /* Returns < 0 if we couldn't compress the data or the
         * compressed result is longer than the original */

        if (src_size < 80)
                return -ENOBUFS;

        ret = sym_lzma_stream_buffer_encode(filters, LZMA_CHECK_NONE, NULL,
                                        src, src_size, dst, &out_pos, dst_alloc_size);
        if (ret != LZMA_OK)
                return -ENOBUFS;

        *dst_size = out_pos;
        return 0;
#else
        return -EPROTONOSUPPORT;
#endif
}

int dlopen_lz4(void) {
#if HAVE_LZ4
        SD_ELF_NOTE_DLOPEN(
                        "lz4",
                        "Support lz4 compression in journal and coredump files",
                        COMPRESSION_PRIORITY_LZ4,
                        "liblz4.so.1");

        return dlopen_many_sym_or_warn(
                        &lz4_dl,
                        "liblz4.so.1", LOG_DEBUG,
                        DLSYM_ARG(LZ4F_compressBegin),
                        DLSYM_ARG(LZ4F_compressBound),
                        DLSYM_ARG(LZ4F_compressEnd),
                        DLSYM_ARG(LZ4F_compressUpdate),
                        DLSYM_ARG(LZ4F_createCompressionContext),
                        DLSYM_ARG(LZ4F_createDecompressionContext),
                        DLSYM_ARG(LZ4F_decompress),
                        DLSYM_ARG(LZ4F_freeCompressionContext),
                        DLSYM_ARG(LZ4F_freeDecompressionContext),
                        DLSYM_ARG(LZ4F_isError),
                        DLSYM_ARG(LZ4_compress_default),
                        DLSYM_ARG(LZ4_compress_HC),
                        DLSYM_ARG(LZ4_decompress_safe),
                        DLSYM_ARG(LZ4_decompress_safe_partial),
                        DLSYM_ARG(LZ4_versionNumber));
#else
        return -EOPNOTSUPP;
#endif
}

int compress_blob_lz4(const void *src, uint64_t src_size,
                      void *dst, size_t dst_alloc_size, size_t *dst_size, int level) {

        assert(src);
        assert(src_size > 0);
        assert(dst);
        assert(dst_alloc_size > 0);
        assert(dst_size);

#if HAVE_LZ4
        int r;

        r = dlopen_lz4();
        if (r < 0)
                return r;
        /* Returns < 0 if we couldn't compress the data or the
         * compressed result is longer than the original */

        if (src_size < 9)
                return -ENOBUFS;

        if (level <= 0)
                r = sym_LZ4_compress_default(src, (char*)dst + 8, src_size, (int) dst_alloc_size - 8);
        else
                r = sym_LZ4_compress_HC(src, (char*)dst + 8, src_size, (int) dst_alloc_size - 8, level);
        if (r <= 0)
                return -ENOBUFS;

        unaligned_write_le64(dst, src_size);
        *dst_size = r + 8;

        return 0;
#else
        return -EPROTONOSUPPORT;
#endif
}

int dlopen_zstd(void) {
#if HAVE_ZSTD
        SD_ELF_NOTE_DLOPEN(
                        "zstd",
                        "Support zstd compression in journal and coredump files",
                        COMPRESSION_PRIORITY_ZSTD,
                        "libzstd.so.1");

        return dlopen_many_sym_or_warn(
                        &zstd_dl,
                        "libzstd.so.1", LOG_DEBUG,
                        DLSYM_ARG(ZSTD_getErrorCode),
                        DLSYM_ARG(ZSTD_compress),
                        DLSYM_ARG(ZSTD_getFrameContentSize),
                        DLSYM_ARG(ZSTD_decompressStream),
                        DLSYM_ARG(ZSTD_getErrorName),
                        DLSYM_ARG(ZSTD_DStreamOutSize),
                        DLSYM_ARG(ZSTD_CStreamInSize),
                        DLSYM_ARG(ZSTD_CStreamOutSize),
                        DLSYM_ARG(ZSTD_CCtx_setParameter),
                        DLSYM_ARG(ZSTD_compressStream2),
                        DLSYM_ARG(ZSTD_DStreamInSize),
                        DLSYM_ARG(ZSTD_freeCCtx),
                        DLSYM_ARG(ZSTD_freeDCtx),
                        DLSYM_ARG(ZSTD_isError),
                        DLSYM_ARG(ZSTD_createDCtx),
                        DLSYM_ARG(ZSTD_createCCtx));
#else
        return -EOPNOTSUPP;
#endif
}

int compress_blob_zstd(
                const void *src, uint64_t src_size,
                void *dst, size_t dst_alloc_size, size_t *dst_size, int level) {

        assert(src);
        assert(src_size > 0);
        assert(dst);
        assert(dst_alloc_size > 0);
        assert(dst_size);

#if HAVE_ZSTD
        size_t k;
        int r;

        r = dlopen_zstd();
        if (r < 0)
                return r;

        k = sym_ZSTD_compress(dst, dst_alloc_size, src, src_size, level < 0 ? 0 : level);
        if (sym_ZSTD_isError(k))
                return zstd_ret_to_errno(k);

        *dst_size = k;
        return 0;
#else
        return -EPROTONOSUPPORT;
#endif
}

int decompress_blob_xz(
                const void *src,
                uint64_t src_size,
                void **dst,
                size_t* dst_size,
                size_t dst_max) {

        assert(src);
        assert(src_size > 0);
        assert(dst);
        assert(dst_size);

#if HAVE_XZ
        int r;

        r = dlopen_lzma();
        if (r < 0)
                return r;

        _cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT;
        lzma_ret ret = sym_lzma_stream_decoder(&s, UINT64_MAX, 0);
        if (ret != LZMA_OK)
                return -ENOMEM;

        size_t space = MIN(src_size * 2, dst_max ?: SIZE_MAX);
        if (!greedy_realloc(dst, space, 1))
                return -ENOMEM;

        s.next_in = src;
        s.avail_in = src_size;

        s.next_out = *dst;
        s.avail_out = space;

        for (;;) {
                size_t used;

                ret = sym_lzma_code(&s, LZMA_FINISH);
                if (ret == LZMA_STREAM_END)
                        break;
                if (ret != LZMA_OK)
                        return -ENOMEM;

                if (dst_max > 0 && (space - s.avail_out) >= dst_max)
                        break;
                if (dst_max > 0 && space == dst_max)
                        return -ENOBUFS;

                used = space - s.avail_out;
                /* Silence static analyzers, space is bounded by allocation size */
                assert(space <= SIZE_MAX / 2);
                space = MIN(2 * space, dst_max ?: SIZE_MAX);
                if (!greedy_realloc(dst, space, 1))
                        return -ENOMEM;

                s.avail_out = space - used;
                s.next_out = *(uint8_t**)dst + used;
        }

        *dst_size = space - s.avail_out;
        return 0;
#else
        return -EPROTONOSUPPORT;
#endif
}

int decompress_blob_lz4(
                const void *src,
                uint64_t src_size,
                void **dst,
                size_t* dst_size,
                size_t dst_max) {

        assert(src);
        assert(src_size > 0);
        assert(dst);
        assert(dst_size);

#if HAVE_LZ4
        char* out;
        int r, size; /* LZ4 uses int for size */

        r = dlopen_lz4();
        if (r < 0)
                return r;

        if (src_size <= 8)
                return -EBADMSG;

        size = unaligned_read_le64(src);
        if (size < 0 || (unsigned) size != unaligned_read_le64(src))
                return -EFBIG;
        out = greedy_realloc(dst, size, 1);
        if (!out)
                return -ENOMEM;

        r = sym_LZ4_decompress_safe((char*)src + 8, out, src_size - 8, size);
        if (r < 0 || r != size)
                return -EBADMSG;

        *dst_size = size;
        return 0;
#else
        return -EPROTONOSUPPORT;
#endif
}

int decompress_blob_zstd(
                const void *src,
                uint64_t src_size,
                void **dst,
                size_t *dst_size,
                size_t dst_max) {

        assert(src);
        assert(src_size > 0);
        assert(dst);
        assert(dst_size);

#if HAVE_ZSTD
        uint64_t size;
        int r;

        r = dlopen_zstd();
        if (r < 0)
                return r;

        size = sym_ZSTD_getFrameContentSize(src, src_size);
        if (IN_SET(size, ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN))
                return -EBADMSG;

        if (dst_max > 0 && size > dst_max)
                size = dst_max;
        if (size > SIZE_MAX)
                return -E2BIG;

        if (!(greedy_realloc(dst, MAX(sym_ZSTD_DStreamOutSize(), size), 1)))
                return -ENOMEM;

        _cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = sym_ZSTD_createDCtx();
        if (!dctx)
                return -ENOMEM;

        ZSTD_inBuffer input = {
                .src = src,
                .size = src_size,
        };
        ZSTD_outBuffer output = {
                .dst = *dst,
                .size = MALLOC_SIZEOF_SAFE(*dst),
        };

        size_t k = sym_ZSTD_decompressStream(dctx, &output, &input);
        if (sym_ZSTD_isError(k))
                return log_debug_errno(zstd_ret_to_errno(k), "ZSTD decoder failed: %s", sym_ZSTD_getErrorName(k));
        if (output.pos < size)
                return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG), "ZSTD decoded less data than indicated, probably corrupted stream.");

        *dst_size = size;
        return 0;
#else
        return -EPROTONOSUPPORT;
#endif
}

int decompress_blob(
                Compression compression,
                const void *src,
                uint64_t src_size,
                void **dst,
                size_t* dst_size,
                size_t dst_max) {

        switch (compression) {
        case COMPRESSION_XZ:
                return decompress_blob_xz(
                                src, src_size,
                                dst, dst_size, dst_max);
        case COMPRESSION_LZ4:
                return decompress_blob_lz4(
                                src, src_size,
                                dst, dst_size, dst_max);
        case COMPRESSION_ZSTD:
                return decompress_blob_zstd(
                                src, src_size,
                                dst, dst_size, dst_max);
        default:
                return -EPROTONOSUPPORT;
        }
}

int decompress_startswith_xz(
                const void *src,
                uint64_t src_size,
                void **buffer,
                const void *prefix,
                size_t prefix_len,
                uint8_t extra) {

        /* Checks whether the decompressed blob starts with the mentioned prefix. The byte extra needs to
         * follow the prefix */

        assert(src);
        assert(src_size > 0);
        assert(buffer);
        assert(prefix);

#if HAVE_XZ
        int r;

        r = dlopen_lzma();
        if (r < 0)
                return r;

        _cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT;
        lzma_ret ret = sym_lzma_stream_decoder(&s, UINT64_MAX, 0);
        if (ret != LZMA_OK)
                return -EBADMSG;

        if (!(greedy_realloc(buffer, ALIGN_8(prefix_len + 1), 1)))
                return -ENOMEM;

        size_t allocated = MALLOC_SIZEOF_SAFE(*buffer);

        s.next_in = src;
        s.avail_in = src_size;

        s.next_out = *buffer;
        s.avail_out = allocated;

        for (;;) {
                ret = sym_lzma_code(&s, LZMA_FINISH);

                if (!IN_SET(ret, LZMA_OK, LZMA_STREAM_END))
                        return -EBADMSG;

                if (allocated - s.avail_out >= prefix_len + 1)
                        return memcmp(*buffer, prefix, prefix_len) == 0 &&
                                ((const uint8_t*) *buffer)[prefix_len] == extra;

                if (ret == LZMA_STREAM_END)
                        return 0;

                s.avail_out += allocated;

                if (!(greedy_realloc(buffer, allocated * 2, 1)))
                        return -ENOMEM;

                allocated = MALLOC_SIZEOF_SAFE(*buffer);
                s.next_out = *(uint8_t**)buffer + allocated - s.avail_out;
        }

#else
        return -EPROTONOSUPPORT;
#endif
}

int decompress_startswith_lz4(
                const void *src,
                uint64_t src_size,
                void **buffer,
                const void *prefix,
                size_t prefix_len,
                uint8_t extra) {

        /* Checks whether the decompressed blob starts with the mentioned prefix. The byte extra needs to
         * follow the prefix */

        assert(src);
        assert(src_size > 0);
        assert(buffer);
        assert(prefix);

#if HAVE_LZ4
        size_t allocated;
        int r;

        r = dlopen_lz4();
        if (r < 0)
                return r;

        if (src_size <= 8)
                return -EBADMSG;

        if (!(greedy_realloc(buffer, ALIGN_8(prefix_len + 1), 1)))
                return -ENOMEM;
        allocated = MALLOC_SIZEOF_SAFE(*buffer);

        r = sym_LZ4_decompress_safe_partial(
                        (char*)src + 8,
                        *buffer,
                        src_size - 8,
                        prefix_len + 1,
                        allocated);

        /* One lz4 < 1.8.3, we might get "failure" (r < 0), or "success" where just a part of the buffer is
         * decompressed. But if we get a smaller amount of bytes than requested, we don't know whether there
         * isn't enough data to fill the requested size or whether we just got a partial answer.
         */
        if (r < 0 || (size_t) r < prefix_len + 1) {
                size_t size;

                if (sym_LZ4_versionNumber() >= 10803)
                        /* We trust that the newer lz4 decompresses the number of bytes we
                         * requested if available in the compressed string. */
                        return 0;

                if (r > 0)
                        /* Compare what we have first, in case of mismatch we can
                         * shortcut the full comparison. */
                        if (memcmp(*buffer, prefix, r) != 0)
                                return 0;

                /* Before version 1.8.3, lz4 always tries to decode full a "sequence",
                 * so in pathological cases might need to decompress the full field. */
                r = decompress_blob_lz4(src, src_size, buffer, &size, 0);
                if (r < 0)
                        return r;

                if (size < prefix_len + 1)
                        return 0;
        }

        return memcmp(*buffer, prefix, prefix_len) == 0 &&
                ((const uint8_t*) *buffer)[prefix_len] == extra;
#else
        return -EPROTONOSUPPORT;
#endif
}

int decompress_startswith_zstd(
                const void *src,
                uint64_t src_size,
                void **buffer,
                const void *prefix,
                size_t prefix_len,
                uint8_t extra) {

        assert(src);
        assert(src_size > 0);
        assert(buffer);
        assert(prefix);

#if HAVE_ZSTD
        int r;

        r = dlopen_zstd();
        if (r < 0)
                return r;

        uint64_t size = sym_ZSTD_getFrameContentSize(src, src_size);
        if (IN_SET(size, ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN))
                return -EBADMSG;

        if (size < prefix_len + 1)
                return 0; /* Decompressed text too short to match the prefix and extra */

        _cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = sym_ZSTD_createDCtx();
        if (!dctx)
                return -ENOMEM;

        if (!(greedy_realloc(buffer, MAX(sym_ZSTD_DStreamOutSize(), prefix_len + 1), 1)))
                return -ENOMEM;

        ZSTD_inBuffer input = {
                .src = src,
                .size = src_size,
        };
        ZSTD_outBuffer output = {
                .dst = *buffer,
                .size = MALLOC_SIZEOF_SAFE(*buffer),
        };
        size_t k;

        k = sym_ZSTD_decompressStream(dctx, &output, &input);
        if (sym_ZSTD_isError(k)) {
                log_debug("ZSTD decoder failed: %s", sym_ZSTD_getErrorName(k));
                return zstd_ret_to_errno(k);
        }
        assert(output.pos >= prefix_len + 1);

        return memcmp(*buffer, prefix, prefix_len) == 0 &&
                ((const uint8_t*) *buffer)[prefix_len] == extra;
#else
        return -EPROTONOSUPPORT;
#endif
}

int decompress_startswith(
                Compression compression,
                const void *src,
                uint64_t src_size,
                void **buffer,
                const void *prefix,
                size_t prefix_len,
                uint8_t extra) {

        switch (compression) {

        case COMPRESSION_XZ:
                return decompress_startswith_xz(
                                src, src_size,
                                buffer,
                                prefix, prefix_len,
                                extra);

        case COMPRESSION_LZ4:
                return decompress_startswith_lz4(
                                src, src_size,
                                buffer,
                                prefix, prefix_len,
                                extra);
        case COMPRESSION_ZSTD:
                return decompress_startswith_zstd(
                                src, src_size,
                                buffer,
                                prefix, prefix_len,
                                extra);
        default:
                return -EBADMSG;
        }
}

int compress_stream_xz(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size) {
        assert(fdf >= 0);
        assert(fdt >= 0);

#if HAVE_XZ
        int r;

        r = dlopen_lzma();
        if (r < 0)
                return r;

        _cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT;
        lzma_ret ret = sym_lzma_easy_encoder(&s, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
        if (ret != LZMA_OK)
                return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
                                       "Failed to initialize XZ encoder: code %u",
                                       ret);

        uint8_t buf[BUFSIZ], out[BUFSIZ];
        lzma_action action = LZMA_RUN;
        for (;;) {
                if (s.avail_in == 0 && action == LZMA_RUN) {
                        size_t m = sizeof(buf);
                        ssize_t n;

                        if (max_bytes != UINT64_MAX && (uint64_t) m > max_bytes)
                                m = (size_t) max_bytes;

                        n = read(fdf, buf, m);
                        if (n < 0)
                                return -errno;
                        if (n == 0)
                                action = LZMA_FINISH;
                        else {
                                s.next_in = buf;
                                s.avail_in = n;

                                if (max_bytes != UINT64_MAX) {
                                        assert(max_bytes >= (uint64_t) n);
                                        max_bytes -= n;
                                }
                        }
                }

                if (s.avail_out == 0) {
                        s.next_out = out;
                        s.avail_out = sizeof(out);
                }

                ret = sym_lzma_code(&s, action);
                if (!IN_SET(ret, LZMA_OK, LZMA_STREAM_END))
                        return log_error_errno(SYNTHETIC_ERRNO(EBADMSG),
                                               "Compression failed: code %u",
                                               ret);

                if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
                        ssize_t n, k;

                        n = sizeof(out) - s.avail_out;

                        k = loop_write(fdt, out, n);
                        if (k < 0)
                                return k;

                        if (ret == LZMA_STREAM_END) {
                                if (ret_uncompressed_size)
                                        *ret_uncompressed_size = s.total_in;

                                if (s.total_in == 0)
                                        log_debug("XZ compression finished (no input data)");
                                else
                                        log_debug("XZ compression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
                                                  s.total_in, s.total_out,
                                                  (double) s.total_out / s.total_in * 100);

                                return 0;
                        }
                }
        }
#else
        return -EPROTONOSUPPORT;
#endif
}

#define LZ4_BUFSIZE (512*1024u)

int compress_stream_lz4(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size) {

#if HAVE_LZ4
        LZ4F_errorCode_t c;
        _cleanup_(LZ4F_freeCompressionContextp) LZ4F_compressionContext_t ctx = NULL;
        _cleanup_free_ void *in_buff = NULL;
        _cleanup_free_ char *out_buff = NULL;
        size_t out_allocsize, n, offset = 0, frame_size;
        uint64_t total_in = 0, total_out;
        int r;
        static const LZ4F_preferences_t preferences = {
                .frameInfo.blockSizeID = 5,
        };

        r = dlopen_lz4();
        if (r < 0)
                return r;

        c = sym_LZ4F_createCompressionContext(&ctx, LZ4F_VERSION);
        if (sym_LZ4F_isError(c))
                return -ENOMEM;

        frame_size = sym_LZ4F_compressBound(LZ4_BUFSIZE, &preferences);
        out_allocsize = frame_size + 64*1024; /* add some space for header and trailer */
        out_buff = malloc(out_allocsize);
        if (!out_buff)
                return -ENOMEM;

        in_buff = malloc(LZ4_BUFSIZE);
        if (!in_buff)
                return -ENOMEM;

        n = offset = total_out = sym_LZ4F_compressBegin(ctx, out_buff, out_allocsize, &preferences);
        if (sym_LZ4F_isError(n))
                return -EINVAL;

        log_debug("Buffer size is %zu bytes, header size %zu bytes.", out_allocsize, n);

        for (;;) {
                ssize_t k;

                k = loop_read(fdf, in_buff, LZ4_BUFSIZE, true);
                if (k < 0)
                        return k;
                if (k == 0)
                        break;
                n = sym_LZ4F_compressUpdate(ctx, out_buff + offset, out_allocsize - offset,
                                        in_buff, k, NULL);
                if (sym_LZ4F_isError(n))
                        return -ENOTRECOVERABLE;

                total_in += k;
                offset += n;
                total_out += n;

                if (max_bytes != UINT64_MAX && total_out > (size_t) max_bytes)
                        return log_debug_errno(SYNTHETIC_ERRNO(EFBIG),
                                               "Compressed stream longer than %" PRIu64 " bytes", max_bytes);

                if (out_allocsize - offset < frame_size + 4) {
                        k = loop_write(fdt, out_buff, offset);
                        if (k < 0)
                                return k;
                        offset = 0;
                }
        }

        n = sym_LZ4F_compressEnd(ctx, out_buff + offset, out_allocsize - offset, NULL);
        if (sym_LZ4F_isError(n))
                return -ENOTRECOVERABLE;

        offset += n;
        total_out += n;
        r = loop_write(fdt, out_buff, offset);
        if (r < 0)
                return r;

        if (ret_uncompressed_size)
                *ret_uncompressed_size = total_in;

        if (total_in == 0)
                log_debug("LZ4 compression finished (no input data)");
        else
                log_debug("LZ4 compression finished (%" PRIu64 " -> %" PRIu64 " bytes, %.1f%%)",
                          total_in, total_out,
                          (double) total_out / total_in * 100);

        return 0;
#else
        return -EPROTONOSUPPORT;
#endif
}

int decompress_stream_xz(int fdf, int fdt, uint64_t max_bytes) {
        assert(fdf >= 0);
        assert(fdt >= 0);

#if HAVE_XZ
        int r;

        r = dlopen_lzma();
        if (r < 0)
                return r;

        _cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT;
        lzma_ret ret = sym_lzma_stream_decoder(&s, UINT64_MAX, 0);
        if (ret != LZMA_OK)
                return log_debug_errno(SYNTHETIC_ERRNO(ENOMEM),
                                       "Failed to initialize XZ decoder: code %u",
                                       ret);

        uint8_t buf[BUFSIZ], out[BUFSIZ];
        lzma_action action = LZMA_RUN;
        for (;;) {
                if (s.avail_in == 0 && action == LZMA_RUN) {
                        ssize_t n;

                        n = read(fdf, buf, sizeof(buf));
                        if (n < 0)
                                return -errno;
                        if (n == 0)
                                action = LZMA_FINISH;
                        else {
                                s.next_in = buf;
                                s.avail_in = n;
                        }
                }

                if (s.avail_out == 0) {
                        s.next_out = out;
                        s.avail_out = sizeof(out);
                }

                ret = sym_lzma_code(&s, action);
                if (!IN_SET(ret, LZMA_OK, LZMA_STREAM_END))
                        return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG),
                                               "Decompression failed: code %u",
                                               ret);

                if (s.avail_out == 0 || ret == LZMA_STREAM_END) {
                        ssize_t n, k;

                        n = sizeof(out) - s.avail_out;

                        if (max_bytes != UINT64_MAX) {
                                if (max_bytes < (uint64_t) n)
                                        return -EFBIG;

                                max_bytes -= n;
                        }

                        k = loop_write(fdt, out, n);
                        if (k < 0)
                                return k;

                        if (ret == LZMA_STREAM_END) {
                                if (s.total_in == 0)
                                        log_debug("XZ decompression finished (no input data)");
                                else
                                        log_debug("XZ decompression finished (%"PRIu64" -> %"PRIu64" bytes, %.1f%%)",
                                                  s.total_in, s.total_out,
                                                  (double) s.total_out / s.total_in * 100);

                                return 0;
                        }
                }
        }
#else
        return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT),
                               "Cannot decompress file. Compiled without XZ support.");
#endif
}

int decompress_stream_lz4(int fdf, int fdt, uint64_t max_bytes) {
#if HAVE_LZ4
        size_t c;
        _cleanup_(LZ4F_freeDecompressionContextp) LZ4F_decompressionContext_t ctx = NULL;
        _cleanup_free_ char *buf = NULL;
        char *src;
        struct stat st;
        int r;
        size_t total_in = 0, total_out = 0;

        r = dlopen_lz4();
        if (r < 0)
                return r;

        c = sym_LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION);
        if (sym_LZ4F_isError(c))
                return -ENOMEM;

        if (fstat(fdf, &st) < 0)
                return log_debug_errno(errno, "fstat() failed: %m");

        if (file_offset_beyond_memory_size(st.st_size))
                return -EFBIG;

        buf = malloc(LZ4_BUFSIZE);
        if (!buf)
                return -ENOMEM;

        src = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fdf, 0);
        if (src == MAP_FAILED)
                return -errno;

        while (total_in < (size_t) st.st_size) {
                size_t produced = LZ4_BUFSIZE;
                size_t used = st.st_size - total_in;

                c = sym_LZ4F_decompress(ctx, buf, &produced, src + total_in, &used, NULL);
                if (sym_LZ4F_isError(c)) {
                        r = -EBADMSG;
                        goto cleanup;
                }

                total_in += used;
                total_out += produced;

                if (max_bytes != UINT64_MAX && total_out > (size_t) max_bytes) {
                        log_debug("Decompressed stream longer than %"PRIu64" bytes", max_bytes);
                        r = -EFBIG;
                        goto cleanup;
                }

                r = loop_write(fdt, buf, produced);
                if (r < 0)
                        goto cleanup;
        }

        if (total_in == 0)
                log_debug("LZ4 decompression finished (no input data)");
        else
                log_debug("LZ4 decompression finished (%zu -> %zu bytes, %.1f%%)",
                          total_in, total_out,
                          (double) total_out / total_in * 100);
        r = 0;
 cleanup:
        munmap(src, st.st_size);
        return r;
#else
        return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT),
                               "Cannot decompress file. Compiled without LZ4 support.");
#endif
}

int compress_stream_zstd(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size) {
        assert(fdf >= 0);
        assert(fdt >= 0);

#if HAVE_ZSTD
        _cleanup_(ZSTD_freeCCtxp) ZSTD_CCtx *cctx = NULL;
        _cleanup_free_ void *in_buff = NULL, *out_buff = NULL;
        size_t in_allocsize, out_allocsize;
        size_t z;
        uint64_t left = max_bytes, in_bytes = 0;
        int r;

        r = dlopen_zstd();
        if (r < 0)
                return r;

        /* Create the context and buffers */
        in_allocsize = sym_ZSTD_CStreamInSize();
        out_allocsize = sym_ZSTD_CStreamOutSize();
        in_buff = malloc(in_allocsize);
        out_buff = malloc(out_allocsize);
        cctx = sym_ZSTD_createCCtx();
        if (!cctx || !out_buff || !in_buff)
                return -ENOMEM;

        z = sym_ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1);
        if (sym_ZSTD_isError(z))
                log_debug("Failed to enable ZSTD checksum, ignoring: %s", sym_ZSTD_getErrorName(z));

        /* This loop read from the input file, compresses that entire chunk,
         * and writes all output produced to the output file.
         */
        for (;;) {
                bool is_last_chunk;
                ZSTD_inBuffer input = {
                        .src = in_buff,
                        .size = 0,
                        .pos = 0
                };
                ssize_t red;

                red = loop_read(fdf, in_buff, in_allocsize, true);
                if (red < 0)
                        return red;
                is_last_chunk = red == 0;

                in_bytes += (size_t) red;
                input.size = (size_t) red;

                for (bool finished = false; !finished;) {
                        ZSTD_outBuffer output = {
                                .dst = out_buff,
                                .size = out_allocsize,
                                .pos = 0
                        };
                        size_t remaining;
                        ssize_t wrote;

                        /* Compress into the output buffer and write all of the
                         * output to the file so we can reuse the buffer next
                         * iteration.
                         */
                        remaining = sym_ZSTD_compressStream2(
                                cctx, &output, &input,
                                is_last_chunk ? ZSTD_e_end : ZSTD_e_continue);

                        if (sym_ZSTD_isError(remaining)) {
                                log_debug("ZSTD encoder failed: %s", sym_ZSTD_getErrorName(remaining));
                                return zstd_ret_to_errno(remaining);
                        }

                        if (left < output.pos)
                                return -EFBIG;

                        wrote = loop_write_full(fdt, output.dst, output.pos, USEC_INFINITY);
                        if (wrote < 0)
                                return wrote;

                        left -= output.pos;

                        /* If we're on the last chunk we're finished when zstd
                         * returns 0, which means its consumed all the input AND
                         * finished the frame. Otherwise, we're finished when
                         * we've consumed all the input.
                         */
                        finished = is_last_chunk ? (remaining == 0) : (input.pos == input.size);
                }

                /* zstd only returns 0 when the input is completely consumed */
                assert(input.pos == input.size);
                if (is_last_chunk)
                        break;
        }

        if (ret_uncompressed_size)
                *ret_uncompressed_size = in_bytes;

        if (in_bytes == 0)
                log_debug("ZSTD compression finished (no input data)");
        else
                log_debug("ZSTD compression finished (%" PRIu64 " -> %" PRIu64 " bytes, %.1f%%)",
                          in_bytes, max_bytes - left, (double) (max_bytes - left) / in_bytes * 100);

        return 0;
#else
        return -EPROTONOSUPPORT;
#endif
}

int decompress_stream_zstd(int fdf, int fdt, uint64_t max_bytes) {
        assert(fdf >= 0);
        assert(fdt >= 0);

#if HAVE_ZSTD
        _cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = NULL;
        _cleanup_free_ void *in_buff = NULL, *out_buff = NULL;
        size_t in_allocsize, out_allocsize;
        size_t last_result = 0;
        uint64_t left = max_bytes, in_bytes = 0;
        int r;

        r = dlopen_zstd();
        if (r < 0)
                return r;
        /* Create the context and buffers */
        in_allocsize = sym_ZSTD_DStreamInSize();
        out_allocsize = sym_ZSTD_DStreamOutSize();
        in_buff = malloc(in_allocsize);
        out_buff = malloc(out_allocsize);
        dctx = sym_ZSTD_createDCtx();
        if (!dctx || !out_buff || !in_buff)
                return -ENOMEM;

        /* This loop assumes that the input file is one or more concatenated
         * zstd streams. This example won't work if there is trailing non-zstd
         * data at the end, but streaming decompression in general handles this
         * case. ZSTD_decompressStream() returns 0 exactly when the frame is
         * completed, and doesn't consume input after the frame.
         */
        for (;;) {
                bool has_error = false;
                ZSTD_inBuffer input = {
                        .src = in_buff,
                        .size = 0,
                        .pos = 0
                };
                ssize_t red;

                red = loop_read(fdf, in_buff, in_allocsize, true);
                if (red < 0)
                        return red;
                if (red == 0)
                        break;

                in_bytes += (size_t) red;
                input.size = (size_t) red;
                input.pos = 0;

                /* Given a valid frame, zstd won't consume the last byte of the
                 * frame until it has flushed all of the decompressed data of
                 * the frame. So input.pos < input.size means frame is not done
                 * or there is still output available.
                 */
                while (input.pos < input.size) {
                        ZSTD_outBuffer output = {
                                .dst = out_buff,
                                .size = out_allocsize,
                                .pos = 0
                        };
                        ssize_t wrote;
                        /* The return code is zero if the frame is complete, but
                         * there may be multiple frames concatenated together.
                         * Zstd will automatically reset the context when a
                         * frame is complete. Still, calling ZSTD_DCtx_reset()
                         * can be useful to reset the context to a clean state,
                         * for instance if the last decompression call returned
                         * an error.
                         */
                        last_result = sym_ZSTD_decompressStream(dctx, &output, &input);
                        if (sym_ZSTD_isError(last_result)) {
                                has_error = true;
                                break;
                        }

                        if (left < output.pos)
                                return -EFBIG;

                        wrote = loop_write_full(fdt, output.dst, output.pos, USEC_INFINITY);
                        if (wrote < 0)
                                return wrote;

                        left -= output.pos;
                }
                if (has_error)
                        break;
        }

        if (in_bytes == 0)
                return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG), "ZSTD decoder failed: no data read");

        if (last_result != 0) {
                /* The last return value from ZSTD_decompressStream did not end
                 * on a frame, but we reached the end of the file! We assume
                 * this is an error, and the input was truncated.
                 */
                log_debug("ZSTD decoder failed: %s", sym_ZSTD_getErrorName(last_result));
                return zstd_ret_to_errno(last_result);
        }

        if (in_bytes == 0)
                log_debug("ZSTD decompression finished (no input data)");
        else
                log_debug("ZSTD decompression finished (%" PRIu64 " -> %" PRIu64 " bytes, %.1f%%)",
                          in_bytes,
                          max_bytes - left,
                          (double) (max_bytes - left) / in_bytes * 100);
        return 0;
#else
        return log_debug_errno(SYNTHETIC_ERRNO(EPROTONOSUPPORT),
                               "Cannot decompress file. Compiled without ZSTD support.");
#endif
}

int decompress_stream(const char *filename, int fdf, int fdt, uint64_t max_bytes) {

        if (endswith(filename, ".lz4"))
                return decompress_stream_lz4(fdf, fdt, max_bytes);
        if (endswith(filename, ".xz"))
                return decompress_stream_xz(fdf, fdt, max_bytes);
        if (endswith(filename, ".zst"))
                return decompress_stream_zstd(fdf, fdt, max_bytes);

        return -EPROTONOSUPPORT;
}

systemd / systemd / 23825567702

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous