20284884426-2753

Committed 16 Dec 2025 10:23PM UTC coverage: 68.23% (-0.7%) from 68.903%

Build # 20284884426-2753

Build Type

push

github

Committed by

tstack

Commit Message

[log] show invalid utf hex dump in log view too

Run Details

25 of 25 new or added lines in 2 files covered. (100.0%)

503 existing lines in 33 files now uncovered.

51170 of 74996 relevant lines covered (68.23%)

433797.6 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.26

/src/logfile.hh

/**
 * Copyright (c) 2007-2012, Timothy Stack
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 * * Neither the name of Timothy Stack nor the names of its contributors
 * may be used to endorse or promote products derived from this software
 * without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * @file logfile.hh
 */

#ifndef logfile_hh
#define logfile_hh

#include <chrono>
#include <filesystem>
#include <string>
#include <utility>
#include <vector>

#include <stdint.h>
#include <stdio.h>
#include <sys/resource.h>
#include <sys/stat.h>
#include <sys/types.h>

#include "ArenaAlloc/arenaalloc.h"
#include "base/auto_fd.hh"
#include "base/auto_mem.hh"
#include "base/lnav_log.hh"
#include "base/map_util.hh"
#include "base/progress.hh"
#include "base/result.h"
#include "bookmarks.hh"
#include "file_options.hh"
#include "line_buffer.hh"
#include "log_format_fwd.hh"
#include "logfile_fwd.hh"
#include "mapbox/variant.hpp"
#include "safe/safe.h"
#include "shared_buffer.hh"
#include "unique_path.hh"

/**
 * Observer interface for logfile indexing progress.
 *
 * @see logfile
 */
class logfile_observer {
public:
    virtual ~logfile_observer() = default;

    /**
     * @param lf The logfile object that is doing the indexing.
     * @param off The current offset in the file being processed.
     * @param total The total size of the file.
     * @return false
     */
    virtual lnav::progress_result_t logfile_indexing(const logfile* lf,
                                                     file_off_t off,
                                                     file_ssize_t total) = 0;
};

struct logfile_activity {
    int64_t la_polls{0};
    int64_t la_reads{0};
    struct rusage la_initial_index_rusage{};
};

/**
 * Container for the lines in a log file and some metadata.
 */
class logfile
    : public unique_path_source
    , public std::enable_shared_from_this<logfile> {
public:
    using iterator = std::vector<logline>::iterator;
    using const_iterator = std::vector<logline>::const_iterator;

    struct metadata {
        text_format_t m_format;
        std::string m_value;
    };

    /**
     * Construct a logfile with the given arguments.
     *
     * @param filename The name of the log file.
     * @param fd The file descriptor for accessing the file or -1 if the
     * constructor should open the file specified by 'filename'.  The
     * descriptor needs to be seekable.
     */
    static Result<std::shared_ptr<logfile>, std::string> open(
        std::filesystem::path filename,
        const logfile_open_options& loo,
        auto_fd fd = auto_fd{});

    ~logfile() override;

    const logfile_activity& get_activity() const { return this->lf_activity; }

    std::optional<std::filesystem::path> get_actual_path() const
    {
        return this->lf_actual_path;
    }

    /** @return The filename as given in the constructor. */
    const std::filesystem::path& get_filename() const
    {
        return this->lf_filename;
    }

    const std::string get_filename_as_string() const
    {
        return this->lf_filename_as_string;
    }

    std::filesystem::path get_path_for_key() const;

    /** @return The filename as given in the constructor, excluding the path
     * prefix. */
    const std::string& get_basename() const { return this->lf_basename; }

    int get_fd() const { return this->lf_line_buffer.get_fd(); }

    /** @param filename The new filename for this log file. */
    void set_filename(const std::string& filename);

    const std::string& get_content_id() const { return this->lf_content_id; }

    /** @return The inode for this log file. */
    const struct stat& get_stat() const { return this->lf_stat; }

    time_t get_origin_mtime() const;

    size_t get_longest_line_length() const { return this->lf_longest_line; }

    bool is_compressed() const { return this->lf_line_buffer.is_compressed(); }

    bool has_line_metadata() const
    {
        return this->lf_line_buffer.has_line_metadata();
    }

    bool is_valid_filename() const { return this->lf_valid_filename; }

    file_off_t get_index_size() const { return this->lf_index_size; }

    /**
     * @return The amount of data in the (possibly compressed) file that has
     * been indexed.
     */
    file_off_t get_indexed_file_offset() const
    {
        return this->lf_line_buffer.get_read_offset(this->lf_index_size);
    }

    int get_index_generation() const { return this->lf_index_generation; }

    file_ssize_t get_content_size() const
    {
        auto lb_size = this->lf_line_buffer.get_file_size();
        if (lb_size != -1) {
            return lb_size;
        }
        return this->lf_stat.st_size;
    }

    std::optional<const_iterator> line_for_offset(file_off_t off) const;

    /**
     * @return The detected format, rebuild_index() must be called before this
     * will return a value other than NULL.
     */
    std::shared_ptr<log_format> get_format() const { return this->lf_format; }

    log_format* get_format_ptr() const { return this->lf_format.get(); }

    intern_string_t get_format_name() const;

    std::optional<text_format_t> get_text_format() const
    {
        return this->lf_text_format;
    }

    void set_text_format(std::optional<text_format_t> tf)
    {
        this->lf_text_format = tf;
    }

    std::chrono::microseconds get_modified_time() const
    {
        return this->lf_index_time;
    }

    int get_time_offset_line() const { return this->lf_time_offset_line; }

    const timeval& get_time_offset() const { return this->lf_time_offset; }

    void adjust_content_time(int line,
                             const struct timeval& tv,
                             bool abs_offset = true);

    void clear_time_offset()
    {
        timeval tv = {0, 0};

        this->adjust_content_time(-1, tv);
    }

    bool mark_as_duplicate(const std::string& name);

    const logfile_open_options& get_open_options() const
    {
        return this->lf_options;
    }

    void set_include_in_session(bool enabled)
    {
        this->lf_options.with_include_in_session(enabled);
    }

    void set_init_location(file_location_t loc)
    {
        this->lf_options.with_init_location(loc);
    }

    void reset_state();

    bool is_time_adjusted() const
    {
        return (this->lf_time_offset.tv_sec != 0
                || this->lf_time_offset.tv_usec != 0);
    }

    iterator begin() { return this->lf_index.begin(); }

    const_iterator begin() const { return this->lf_index.begin(); }

    const_iterator cbegin() const { return this->lf_index.begin(); }

    iterator end() { return this->lf_index.end(); }

    const_iterator end() const { return this->lf_index.end(); }

    const_iterator cend() const { return this->lf_index.end(); }

    /** @return The number of lines in the index. */
    size_t size() const { return this->lf_index.size(); }

    std::optional<const_iterator> find_from_time(
        const struct timeval& tv) const;

    logline& operator[](int index) { return this->lf_index[index]; }

    logline& front() { return this->lf_index.front(); }

    logline& back() { return this->lf_index.back(); }

    bool in_range() const;

    /** @return True if this log file still exists. */
    bool exists() const;

    void close() { this->lf_is_closed = true; }

    bool is_closed() const { return this->lf_is_closed; }

    timeval original_line_time(iterator ll);

    Result<shared_buffer_ref, std::string> read_line(iterator ll,
                                                     subline_options opts = {});

    enum class read_format_t {
        plain,
        with_framing,
    };

    struct read_file_result {
        file_range rfr_range;
        std::string rfr_content;
    };

    Result<read_file_result, std::string> read_file(read_format_t format);

    Result<shared_buffer_ref, std::string> read_range(const file_range& fr);

    iterator line_base(iterator ll)
    {
        auto retval = ll;

        while (retval != this->begin() && retval->get_sub_offset() != 0) {
            --retval;
        }

        return retval;
    }

    iterator message_start(iterator ll)
    {
        auto retval = ll;

        while (retval != this->begin()
               && (retval->get_sub_offset() != 0 || !retval->is_message()))
        {
            --retval;
        }

        return retval;
    }

    struct message_length_result {
        file_ssize_t mlr_length;
        size_t mlr_line_count;
        file_range::metadata mlr_metadata;
    };

    message_length_result message_byte_length(const_iterator ll,
                                              bool include_continues = true);

    file_range get_file_range(const_iterator ll, bool include_continues = true)
    {
        auto mlr = this->message_byte_length(ll, include_continues);

        return {
            ll->get_offset(),
            mlr.mlr_length,
            mlr.mlr_metadata,
        };
    }

    file_off_t get_line_content_offset(const_iterator ll)
    {
        return ll->get_offset() + (this->lf_line_buffer.is_piper() ? 22 : 0);
    }

    void read_full_message(const_iterator ll,
                           shared_buffer_ref& msg_out,
                           line_buffer::scan_direction dir
                           = line_buffer::scan_direction::forward,
                           read_format_t format = read_format_t::plain);

    Result<shared_buffer_ref, std::string> read_raw_message(const_iterator ll);

    enum class rebuild_result_t {
        INVALID,
        NO_NEW_LINES,
        NEW_LINES,
        NEW_ORDER,
    };

    /**
     * Index any new data in the log file.
     *
     * @param lo The observer object that will be called regularly during
     * indexing.
     * @return True if any new lines were indexed.
     */
    rebuild_result_t rebuild_index(std::optional<ui_clock::time_point> deadline
                                   = std::nullopt);

    void reobserve_from(iterator iter);

    void set_logfile_observer(logfile_observer* lo)
    {
        this->lf_logfile_observer = lo;
    }

    void set_logline_observer(logline_observer* llo);

    logline_observer* get_logline_observer() const
    {
        return this->lf_logline_observer;
    }

    bool operator<(const logfile& rhs) const
    {
        bool retval;

        if (this->lf_index.empty()) {
            retval = true;
        } else if (rhs.lf_index.empty()) {
            retval = false;
        } else {
            retval = this->lf_index[0] < rhs.lf_index[0];
        }

        return retval;
    }

    bool is_indexing() const { return this->lf_indexing; }

    void set_indexing(bool val) { this->lf_indexing = val; }

    /** Check the invariants for this object. */
    bool invariant()
    {
        require(!this->lf_filename.empty());

        return true;
    }

    std::filesystem::path get_path() const override;

    enum class note_type {
        indexing_disabled,
        duplicate,
        not_utf,
    };

    using note_map = lnav::map::small<note_type, lnav::console::user_message>;
    using safe_notes = safe::Safe<note_map>;

    note_map get_notes() const { return *this->lf_notes.readAccess(); }

    const std::vector<logline_value_stats>& get_value_stats() const
    {
        return this->lf_value_stats;
    }

    const logline_value_stats* stats_for_value(intern_string_t name) const;

    log_format_file_state get_format_file_state() const
    {
        return {
            this->lf_value_stats,
            this->lf_pattern_locks,
        };
    }

    using safe_opid_state = safe::Safe<log_opid_state>;

    safe_opid_state& get_opids() { return this->lf_opids; }

    using safe_thread_id_state = safe::Safe<log_thread_id_state>;

    safe_thread_id_state& get_thread_ids() { return this->lf_thread_ids; }

    void set_logline_opid(uint32_t line_number, string_fragment opid);

    void set_opid_description(string_fragment opid, string_fragment desc);

    void clear_logline_opid(uint32_t line_number);

    void quiesce() { this->lf_line_buffer.quiesce(); }

    void enable_cache() { this->lf_line_buffer.enable_cache(); }

    void dump_stats();

    robin_hood::unordered_map<uint32_t, bookmark_metadata>&
    get_bookmark_metadata()
    {
        return this->lf_bookmark_metadata;
    }

    std::map<std::string, metadata>& get_embedded_metadata()
    {
        return this->lf_embedded_metadata;
    }

    const std::map<std::string, metadata>& get_embedded_metadata() const
    {
        return this->lf_embedded_metadata;
    }

    std::optional<std::pair<std::string, lnav::file_options>> get_file_options()
        const
    {
        return this->lf_file_options;
    }

    const robin_hood::unordered_set<intern_string_t, intern_hasher>&
    get_mismatched_formats() const
    {
        return this->lf_mismatched_formats;
    }

    const std::vector<lnav::console::user_message>& get_format_match_messages()
        const
    {
        return this->lf_format_match_messages;
    }

    struct invalid_line_info {
        static constexpr size_t MAX_INVALID_LINES = 5;

        std::vector<size_t> ili_lines;
        size_t ili_total{0};
    };

    const invalid_line_info& get_invalid_line_info() const
    {
        return this->lf_invalid_lines;
    }

    size_t estimated_remaining_lines() const;

    const std::string& get_decompress_error() const
    {
        return this->lf_line_buffer.get_decompress_error();
    }

    time_range get_content_time_range() const;

    const log_level_stats& get_level_stats() const
    {
        return this->lf_level_stats;
    }

protected:
    /**
     * Process a line from the file.
     *
     * @param offset The offset of the line in the file.
     * @param prefix The contents of the line.
     * @param len The length of the 'prefix' string.
     */
    bool process_prefix(shared_buffer_ref& sbr,
                        const line_info& li,
                        scan_batch_context& sbc);

    void set_format_base_time(log_format* lf, const line_info& li);

private:
    logfile(std::filesystem::path filename, const logfile_open_options& loo);

    bool file_options_have_changed();

    std::filesystem::path lf_filename;
    std::string lf_filename_as_string;
    logfile_open_options lf_options;
    logfile_activity lf_activity;
    bool lf_named_file{true};
    bool lf_valid_filename{true};
    std::optional<std::filesystem::path> lf_actual_path;
    std::string lf_basename;
    std::string lf_content_id;
    struct stat lf_stat{};
    std::shared_ptr<log_format> lf_format;
    uint32_t lf_format_quality{0};
    std::vector<logline> lf_index;
    std::chrono::microseconds lf_index_time{0};
    file_off_t lf_index_size{0};
    int lf_index_generation{0};
    bool lf_sort_needed{false};
    line_buffer lf_line_buffer;
    int lf_time_offset_line{0};
    timeval lf_time_offset{0, 0};
    bool lf_is_closed{false};
    bool lf_indexing{true};
    bool lf_partial_line{false};
    bool lf_zoned_to_local_state{true};
    robin_hood::unordered_set<string_fragment,
                              frag_hasher,
                              std::equal_to<string_fragment>>
        lf_invalidated_opids;
    logline_observer* lf_logline_observer{nullptr};
    logfile_observer* lf_logfile_observer{nullptr};
    size_t lf_longest_line{0};
    std::optional<text_format_t> lf_text_format;
    uint32_t lf_out_of_time_order_count{0};
    safe_notes lf_notes;
    std::vector<logline_value_stats> lf_value_stats;
    log_level_stats lf_level_stats;
    pattern_locks lf_pattern_locks;
    safe_opid_state lf_opids;
    safe_thread_id_state lf_thread_ids;
    size_t lf_watch_count{0};
    ArenaAlloc::Alloc<char> lf_allocator{64 * 1024};
    std::optional<time_t> lf_cached_base_time;
    std::optional<tm> lf_cached_base_tm;

    std::optional<std::pair<file_off_t, size_t>> lf_next_line_cache;
    robin_hood::unordered_set<intern_string_t, intern_hasher>
        lf_mismatched_formats;
    robin_hood::unordered_map<uint32_t, bookmark_metadata> lf_bookmark_metadata;

    std::vector<std::shared_ptr<format_tag_def>> lf_applicable_taggers;
    std::vector<std::shared_ptr<format_partition_def>>
        lf_applicable_partitioners;
    std::map<std::string, metadata> lf_embedded_metadata;
    size_t lf_file_options_generation{0};
    std::optional<std::pair<std::string, lnav::file_options>> lf_file_options;
    std::vector<lnav::console::user_message> lf_format_match_messages;
    invalid_line_info lf_invalid_lines;
    auto_buffer lf_plain_msg_buffer = auto_buffer::alloc(256);
    shared_buffer lf_plain_msg_shared;

    struct content_map_entry {
        file_range cme_range;
        std::chrono::microseconds cme_time;
    };
    file_size_t lf_file_size_at_map_time{0};
    std::vector<content_map_entry> lf_content_map;
    std::optional<content_map_entry> lf_lower_bound_entry;
    std::optional<content_map_entry> lf_upper_bound_entry;
    std::optional<file_size_t> lf_upper_bound_size;

    struct map_read_upper_bound {};
    struct map_read_lower_bound {
        std::chrono::microseconds mrlb_time;
    };
    using map_read_requirement
        = mapbox::util::variant<map_read_upper_bound, map_read_lower_bound>;

    struct map_entry_not_found {};
    struct map_entry_found {
        content_map_entry mef_entry;
    };
    using map_entry_result
        = mapbox::util::variant<map_entry_not_found, map_entry_found>;

    map_entry_result find_content_map_entry(file_off_t offset,
                                            map_read_requirement req);

    rebuild_result_t build_content_map();
};

class logline_observer {
public:
    virtual ~logline_observer() = default;

    virtual void logline_clear(const logfile& lf) = 0;

    virtual void logline_restart(const logfile& lf, file_size_t rollback_size)
        = 0;

    virtual bool logline_new_lines(const logfile& lf,
                                   logfile::const_iterator ll_begin,
                                   logfile::const_iterator ll_end,
                                   const shared_buffer_ref& sbr) = 0;

    virtual void logline_eof(const logfile& lf) = 0;
};

#endif

1	/**
2	* Copyright (c) 2007-2012, Timothy Stack
3	*
4	* All rights reserved.
5	*
6	* Redistribution and use in source and binary forms, with or without
7	* modification, are permitted provided that the following conditions are met:
8	*
9	* * Redistributions of source code must retain the above copyright notice, this
10	* list of conditions and the following disclaimer.
11	* * Redistributions in binary form must reproduce the above copyright notice,
12	* this list of conditions and the following disclaimer in the documentation
13	* and/or other materials provided with the distribution.
14	* * Neither the name of Timothy Stack nor the names of its contributors
15	* may be used to endorse or promote products derived from this software
16	* without specific prior written permission.
17	*
18	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ''AS IS'' AND ANY
19	* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20	* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21	* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
22	* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23	* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
25	* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27	* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28	*
29	* @file logfile.hh
30	*/
31
32	#ifndef logfile_hh
33	#define logfile_hh
34
35	#include <chrono>
36	#include <filesystem>
37	#include <string>
38	#include <utility>
39	#include <vector>
40
41	#include <stdint.h>
42	#include <stdio.h>
43	#include <sys/resource.h>
44	#include <sys/stat.h>
45	#include <sys/types.h>
46
47	#include "ArenaAlloc/arenaalloc.h"
48	#include "base/auto_fd.hh"
49	#include "base/auto_mem.hh"
50	#include "base/lnav_log.hh"
51	#include "base/map_util.hh"
52	#include "base/progress.hh"
53	#include "base/result.h"
54	#include "bookmarks.hh"
55	#include "file_options.hh"
56	#include "line_buffer.hh"
57	#include "log_format_fwd.hh"
58	#include "logfile_fwd.hh"
59	#include "mapbox/variant.hpp"
60	#include "safe/safe.h"
61	#include "shared_buffer.hh"
62	#include "unique_path.hh"
63
64	/**
65	* Observer interface for logfile indexing progress.
66	*
67	* @see logfile
68	*/
69	class logfile_observer {
70	public:
71	virtual ~logfile_observer() = default;	616✔
72
73	/**
74	* @param lf The logfile object that is doing the indexing.
75	* @param off The current offset in the file being processed.
76	* @param total The total size of the file.
77	* @return false
78	*/
79	virtual lnav::progress_result_t logfile_indexing(const logfile* lf,
80	file_off_t off,
81	file_ssize_t total) = 0;
82	};
83
84	struct logfile_activity {
85	int64_t la_polls{0};
86	int64_t la_reads{0};
87	struct rusage la_initial_index_rusage{};
88	};
89
90	/**
91	* Container for the lines in a log file and some metadata.
92	*/
93	class logfile
94	: public unique_path_source
95	, public std::enable_shared_from_this<logfile> {
96	public:
97	using iterator = std::vector<logline>::iterator;
98	using const_iterator = std::vector<logline>::const_iterator;
99
100	struct metadata {
101	text_format_t m_format;
102	std::string m_value;
103	};
104
105	/**
106	* Construct a logfile with the given arguments.
107	*
108	* @param filename The name of the log file.
109	* @param fd The file descriptor for accessing the file or -1 if the
110	* constructor should open the file specified by 'filename'. The
111	* descriptor needs to be seekable.
112	*/
113	static Result<std::shared_ptr<logfile>, std::string> open(
114	std::filesystem::path filename,
115	const logfile_open_options& loo,
116	auto_fd fd = auto_fd{});
117
118	~logfile() override;
119
120	const logfile_activity& get_activity() const { return this->lf_activity; }
121
122	std::optional<std::filesystem::path> get_actual_path() const	1,871✔
123	{
124	return this->lf_actual_path;	1,871✔
125	}
126
127	/** @return The filename as given in the constructor. */
128	const std::filesystem::path& get_filename() const	12,337✔
129	{
130	return this->lf_filename;	12,337✔
131	}
132
133	const std::string get_filename_as_string() const	556✔
134	{
135	return this->lf_filename_as_string;	556✔
136	}
137
138	std::filesystem::path get_path_for_key() const;
139
140	/** @return The filename as given in the constructor, excluding the path
141	* prefix. */
142	const std::string& get_basename() const { return this->lf_basename; }	15✔
143
144	int get_fd() const { return this->lf_line_buffer.get_fd(); }	644✔
145
146	/** @param filename The new filename for this log file. */
147	void set_filename(const std::string& filename);
148
149	const std::string& get_content_id() const { return this->lf_content_id; }	1,152✔
150
151	/** @return The inode for this log file. */
152	const struct stat& get_stat() const { return this->lf_stat; }	1,560✔
153
154	time_t get_origin_mtime() const;
155
156	size_t get_longest_line_length() const { return this->lf_longest_line; }	1,427✔
157
158	bool is_compressed() const { return this->lf_line_buffer.is_compressed(); }	5,806✔
159
160	bool has_line_metadata() const	15,083✔
161	{
162	return this->lf_line_buffer.has_line_metadata();	15,083✔
163	}
164
165	bool is_valid_filename() const { return this->lf_valid_filename; }	299✔
166
167	file_off_t get_index_size() const { return this->lf_index_size; }	830✔
168
169	/**
170	* @return The amount of data in the (possibly compressed) file that has
171	* been indexed.
172	*/
173	file_off_t get_indexed_file_offset() const
174	{
175	return this->lf_line_buffer.get_read_offset(this->lf_index_size);
176	}
177
178	int get_index_generation() const { return this->lf_index_generation; }
179
180	file_ssize_t get_content_size() const	18,782✔
181	{
182	auto lb_size = this->lf_line_buffer.get_file_size();	18,782✔
183	if (lb_size != -1) {	18,782✔
184	return lb_size;	49✔
185	}
186	return this->lf_stat.st_size;	18,733✔
187	}
188
189	std::optional<const_iterator> line_for_offset(file_off_t off) const;
190
191	/**
192	* @return The detected format, rebuild_index() must be called before this
193	* will return a value other than NULL.
194	*/
195	std::shared_ptr<log_format> get_format() const { return this->lf_format; }	20,647✔
196
197	log_format* get_format_ptr() const { return this->lf_format.get(); }	36,598✔
198
199	intern_string_t get_format_name() const;
200
201	std::optional<text_format_t> get_text_format() const	14,319✔
202	{
203	return this->lf_text_format;	14,319✔
204	}
205
206	void set_text_format(std::optional<text_format_t> tf)
207	{
208	this->lf_text_format = tf;
209	}
210
UNCOV 211	std::chrono::microseconds get_modified_time() const	×
212	{
UNCOV 213	return this->lf_index_time;	×
214	}
215
216	int get_time_offset_line() const { return this->lf_time_offset_line; }	3✔
217
218	const timeval& get_time_offset() const { return this->lf_time_offset; }	24✔
219
220	void adjust_content_time(int line,
221	const struct timeval& tv,
222	bool abs_offset = true);
223
224	void clear_time_offset()	9✔
225	{
226	timeval tv = {0, 0};	9✔
227
228	this->adjust_content_time(-1, tv);	9✔
229	}	9✔
230
231	bool mark_as_duplicate(const std::string& name);
232
233	const logfile_open_options& get_open_options() const	2,082✔
234	{
235	return this->lf_options;	2,082✔
236	}
237
238	void set_include_in_session(bool enabled)	523✔
239	{
240	this->lf_options.with_include_in_session(enabled);	523✔
241	}	523✔
242
243	void set_init_location(file_location_t loc)	×
244	{
245	this->lf_options.with_init_location(loc);	×
246	}
247
248	void reset_state();
249
250	bool is_time_adjusted() const	6,761✔
251	{
252	return (this->lf_time_offset.tv_sec != 0	6,761✔
253	\|\| this->lf_time_offset.tv_usec != 0);	6,761✔
254	}
255
256	iterator begin() { return this->lf_index.begin(); }	527,519✔
257
258	const_iterator begin() const { return this->lf_index.begin(); }	19,299✔
259
260	const_iterator cbegin() const { return this->lf_index.begin(); }	620✔
261
262	iterator end() { return this->lf_index.end(); }	180,572✔
263
264	const_iterator end() const { return this->lf_index.end(); }
265
266	const_iterator cend() const { return this->lf_index.end(); }	×
267
268	/** @return The number of lines in the index. */
269	size_t size() const { return this->lf_index.size(); }	64,990✔
270
271	std::optional<const_iterator> find_from_time(
272	const struct timeval& tv) const;
273
274	logline& operator[](int index) { return this->lf_index[index]; }	4✔
275
276	logline& front() { return this->lf_index.front(); }	2✔
277
278	logline& back() { return this->lf_index.back(); }	14,528✔
279
280	bool in_range() const;
281
282	/** @return True if this log file still exists. */
283	bool exists() const;
284
285	void close() { this->lf_is_closed = true; }	621✔
286
287	bool is_closed() const { return this->lf_is_closed; }	5,789✔
288
289	timeval original_line_time(iterator ll);
290
291	Result<shared_buffer_ref, std::string> read_line(iterator ll,
292	subline_options opts = {});
293
294	enum class read_format_t {
295	plain,
296	with_framing,
297	};
298
299	struct read_file_result {
300	file_range rfr_range;
301	std::string rfr_content;
302	};
303
304	Result<read_file_result, std::string> read_file(read_format_t format);
305
306	Result<shared_buffer_ref, std::string> read_range(const file_range& fr);
307
308	iterator line_base(iterator ll)
309	{
310	auto retval = ll;
311
312	while (retval != this->begin() && retval->get_sub_offset() != 0) {
313	--retval;
314	}
315
316	return retval;
317	}
318
319	iterator message_start(iterator ll)	26✔
320	{
321	auto retval = ll;	26✔
322
323	while (retval != this->begin()	30✔
324	&& (retval->get_sub_offset() != 0 \|\| !retval->is_message()))	30✔
325	{
326	--retval;	4✔
327	}
328
329	return retval;	26✔
330	}
331
332	struct message_length_result {
333	file_ssize_t mlr_length;
334	size_t mlr_line_count;
335	file_range::metadata mlr_metadata;
336	};
337
338	message_length_result message_byte_length(const_iterator ll,
339	bool include_continues = true);
340
341	file_range get_file_range(const_iterator ll, bool include_continues = true)	36,200✔
342	{
343	auto mlr = this->message_byte_length(ll, include_continues);	36,200✔
344
345	return {
346	ll->get_offset(),	36,200✔
347	mlr.mlr_length,	36,200✔
348	mlr.mlr_metadata,
349	};	36,200✔
350	}
351
352	file_off_t get_line_content_offset(const_iterator ll)	822✔
353	{
354	return ll->get_offset() + (this->lf_line_buffer.is_piper() ? 22 : 0);	822✔
355	}
356
357	void read_full_message(const_iterator ll,
358	shared_buffer_ref& msg_out,
359	line_buffer::scan_direction dir
360	= line_buffer::scan_direction::forward,
361	read_format_t format = read_format_t::plain);
362
363	Result<shared_buffer_ref, std::string> read_raw_message(const_iterator ll);
364
365	enum class rebuild_result_t {
366	INVALID,
367	NO_NEW_LINES,
368	NEW_LINES,
369	NEW_ORDER,
370	};
371
372	/**
373	* Index any new data in the log file.
374	*
375	* @param lo The observer object that will be called regularly during
376	* indexing.
377	* @return True if any new lines were indexed.
378	*/
379	rebuild_result_t rebuild_index(std::optional<ui_clock::time_point> deadline
380	= std::nullopt);
381
382	void reobserve_from(iterator iter);
383
384	void set_logfile_observer(logfile_observer* lo)	621✔
385	{
386	this->lf_logfile_observer = lo;	621✔
387	}	621✔
388
389	void set_logline_observer(logline_observer* llo);
390
391	logline_observer* get_logline_observer() const	10,551✔
392	{
393	return this->lf_logline_observer;	10,551✔
394	}
395
396	bool operator<(const logfile& rhs) const
397	{
398	bool retval;
399
400	if (this->lf_index.empty()) {
401	retval = true;
402	} else if (rhs.lf_index.empty()) {
403	retval = false;
404	} else {
405	retval = this->lf_index[0] < rhs.lf_index[0];
406	}
407
408	return retval;
409	}
410
411	bool is_indexing() const { return this->lf_indexing; }	1,172✔
412
413	void set_indexing(bool val) { this->lf_indexing = val; }	6✔
414
415	/** Check the invariants for this object. */
416	bool invariant()	706✔
417	{
418	require(!this->lf_filename.empty());	706✔
419
420	return true;	706✔
421	}
422
423	std::filesystem::path get_path() const override;
424
425	enum class note_type {
426	indexing_disabled,
427	duplicate,
428	not_utf,
429	};
430
431	using note_map = lnav::map::small<note_type, lnav::console::user_message>;
432	using safe_notes = safe::Safe<note_map>;
433
434	note_map get_notes() const { return *this->lf_notes.readAccess(); }	9,990✔
435
436	const std::vector<logline_value_stats>& get_value_stats() const
437	{
438	return this->lf_value_stats;
439	}
440
441	const logline_value_stats* stats_for_value(intern_string_t name) const;
442
443	log_format_file_state get_format_file_state() const	8,542✔
444	{
445	return {
446	this->lf_value_stats,	8,542✔
447	this->lf_pattern_locks,	8,542✔
448	};	8,542✔
449	}
450
451	using safe_opid_state = safe::Safe<log_opid_state>;
452
453	safe_opid_state& get_opids() { return this->lf_opids; }	43✔
454
455	using safe_thread_id_state = safe::Safe<log_thread_id_state>;
456
457	safe_thread_id_state& get_thread_ids() { return this->lf_thread_ids; }	38✔
458
459	void set_logline_opid(uint32_t line_number, string_fragment opid);
460
461	void set_opid_description(string_fragment opid, string_fragment desc);
462
463	void clear_logline_opid(uint32_t line_number);
464
465	void quiesce() { this->lf_line_buffer.quiesce(); }	42✔
466
467	void enable_cache() { this->lf_line_buffer.enable_cache(); }	231✔
468
469	void dump_stats();
470
471	robin_hood::unordered_map<uint32_t, bookmark_metadata>&
472	get_bookmark_metadata()	26,215✔
473	{
474	return this->lf_bookmark_metadata;	26,215✔
475	}
476
477	std::map<std::string, metadata>& get_embedded_metadata()	31✔
478	{
479	return this->lf_embedded_metadata;	31✔
480	}
481
482	const std::map<std::string, metadata>& get_embedded_metadata() const
483	{
484	return this->lf_embedded_metadata;
485	}
486
487	std::optional<std::pair<std::string, lnav::file_options>> get_file_options()	37,026✔
488	const
489	{
490	return this->lf_file_options;	37,026✔
491	}
492
493	const robin_hood::unordered_set<intern_string_t, intern_hasher>&
494	get_mismatched_formats() const
495	{
496	return this->lf_mismatched_formats;
497	}
498
UNCOV 499	const std::vector<lnav::console::user_message>& get_format_match_messages()	×
500	const
501	{
UNCOV 502	return this->lf_format_match_messages;	×
503	}
504
505	struct invalid_line_info {
506	static constexpr size_t MAX_INVALID_LINES = 5;
507
508	std::vector<size_t> ili_lines;
509	size_t ili_total{0};
510	};
511
UNCOV 512	const invalid_line_info& get_invalid_line_info() const	×
513	{
UNCOV 514	return this->lf_invalid_lines;	×
515	}
516
517	size_t estimated_remaining_lines() const;
518
519	const std::string& get_decompress_error() const	900✔
520	{
521	return this->lf_line_buffer.get_decompress_error();	900✔
522	}
523
524	time_range get_content_time_range() const;
525
526	const log_level_stats& get_level_stats() const	26✔
527	{
528	return this->lf_level_stats;	26✔
529	}
530
531	protected:
532	/**
533	* Process a line from the file.
534	*
535	* @param offset The offset of the line in the file.
536	* @param prefix The contents of the line.
537	* @param len The length of the 'prefix' string.
538	*/
539	bool process_prefix(shared_buffer_ref& sbr,
540	const line_info& li,
541	scan_batch_context& sbc);
542
543	void set_format_base_time(log_format* lf, const line_info& li);
544
545	private:
546	logfile(std::filesystem::path filename, const logfile_open_options& loo);
547
548	bool file_options_have_changed();
549
550	std::filesystem::path lf_filename;
551	std::string lf_filename_as_string;
552	logfile_open_options lf_options;
553	logfile_activity lf_activity;
554	bool lf_named_file{true};
555	bool lf_valid_filename{true};
556	std::optional<std::filesystem::path> lf_actual_path;
557	std::string lf_basename;
558	std::string lf_content_id;
559	struct stat lf_stat{};
560	std::shared_ptr<log_format> lf_format;
561	uint32_t lf_format_quality{0};
562	std::vector<logline> lf_index;
563	std::chrono::microseconds lf_index_time{0};	706✔
564	file_off_t lf_index_size{0};
565	int lf_index_generation{0};
566	bool lf_sort_needed{false};
567	line_buffer lf_line_buffer;
568	int lf_time_offset_line{0};
569	timeval lf_time_offset{0, 0};
570	bool lf_is_closed{false};
571	bool lf_indexing{true};
572	bool lf_partial_line{false};
573	bool lf_zoned_to_local_state{true};
574	robin_hood::unordered_set<string_fragment,
575	frag_hasher,
576	std::equal_to<string_fragment>>
577	lf_invalidated_opids;
578	logline_observer* lf_logline_observer{nullptr};
579	logfile_observer* lf_logfile_observer{nullptr};
580	size_t lf_longest_line{0};
581	std::optional<text_format_t> lf_text_format;
582	uint32_t lf_out_of_time_order_count{0};
583	safe_notes lf_notes;
584	std::vector<logline_value_stats> lf_value_stats;
585	log_level_stats lf_level_stats;
586	pattern_locks lf_pattern_locks;
587	safe_opid_state lf_opids;
588	safe_thread_id_state lf_thread_ids;
589	size_t lf_watch_count{0};
590	ArenaAlloc::Alloc<char> lf_allocator{64 * 1024};
591	std::optional<time_t> lf_cached_base_time;
592	std::optional<tm> lf_cached_base_tm;
593
594	std::optional<std::pair<file_off_t, size_t>> lf_next_line_cache;
595	robin_hood::unordered_set<intern_string_t, intern_hasher>
596	lf_mismatched_formats;
597	robin_hood::unordered_map<uint32_t, bookmark_metadata> lf_bookmark_metadata;
598
599	std::vector<std::shared_ptr<format_tag_def>> lf_applicable_taggers;
600	std::vector<std::shared_ptr<format_partition_def>>
601	lf_applicable_partitioners;
602	std::map<std::string, metadata> lf_embedded_metadata;
603	size_t lf_file_options_generation{0};
604	std::optional<std::pair<std::string, lnav::file_options>> lf_file_options;
605	std::vector<lnav::console::user_message> lf_format_match_messages;
606	invalid_line_info lf_invalid_lines;
607	auto_buffer lf_plain_msg_buffer = auto_buffer::alloc(256);
608	shared_buffer lf_plain_msg_shared;
609
610	struct content_map_entry {
611	file_range cme_range;
612	std::chrono::microseconds cme_time;
613	};
614	file_size_t lf_file_size_at_map_time{0};
615	std::vector<content_map_entry> lf_content_map;
616	std::optional<content_map_entry> lf_lower_bound_entry;
617	std::optional<content_map_entry> lf_upper_bound_entry;
618	std::optional<file_size_t> lf_upper_bound_size;
619
620	struct map_read_upper_bound {};
621	struct map_read_lower_bound {
622	std::chrono::microseconds mrlb_time;
623	};
624	using map_read_requirement
625	= mapbox::util::variant<map_read_upper_bound, map_read_lower_bound>;
626
627	struct map_entry_not_found {};
628	struct map_entry_found {
629	content_map_entry mef_entry;
630	};
631	using map_entry_result
632	= mapbox::util::variant<map_entry_not_found, map_entry_found>;
633
634	map_entry_result find_content_map_entry(file_off_t offset,
635	map_read_requirement req);
636
637	rebuild_result_t build_content_map();
638	};
639
640	class logline_observer {
641	public:
642	virtual ~logline_observer() = default;	1,140✔
643
644	virtual void logline_clear(const logfile& lf) = 0;
645
646	virtual void logline_restart(const logfile& lf, file_size_t rollback_size)
647	= 0;
648
649	virtual bool logline_new_lines(const logfile& lf,
650	logfile::const_iterator ll_begin,
651	logfile::const_iterator ll_end,
652	const shared_buffer_ref& sbr) = 0;
653
654	virtual void logline_eof(const logfile& lf) = 0;
655	};
656
657	#endif

tstack / lnav / 20284884426-2753

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous