10831268990

Committed 12 Sep 2024 12:56PM UTC coverage: 77.5% (-11.6%) from 89.14%

Build # 10831268990

Build Type

Pull #85

github

Committed by

chirino

Commit Message

fix: log entries were not being loaded

Broke in ec18c3c63

Signed-off-by: Hiram Chirino <hiram@hiramchirino.com>

Pull Request Pull Request #85: feat: Fast stdin display, lower memory usage, follow/reverse mode, home/end key support. long help support

Run Details

334 of 528 new or added lines in 17 files covered. (63.26%)

15 existing lines in 6 files now uncovered.

1054 of 1360 relevant lines covered (77.5%)

1114.74 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

66.67

/internal/pkg/source/source.go

package source

import (
        "bufio"
        "bytes"
        "errors"
        "io"
        "os"

        "github.com/hedhyw/semerr/pkg/v1/semerr"

        "github.com/hedhyw/json-log-viewer/internal/pkg/config"
)

const (
        maxLineSize = 8 * 1024 * 1024
)

type Source struct {
        // Seeker is used to do random access reads from the file.
        Seeker *os.File
        // Reader is used to read the file sequentially.
        reader *bufio.Reader
        // The log file we are reading from, or a temp file we are writing to (depending on if created with File or Reader func).
        file *os.File
        // offset is the next offset a long entry will be read from.
        offset int64
        // prevFollowSize is the size of the file the last time we checked
        prevFollowSize int64
        // name is the name of the file we are reading.
        name string
        // maxSize is the maximum size of the file we will read.
        maxSize int64
}

func (is *Source) Close() (err error) {
        err = is.file.Close()
        e := is.Seeker.Close()
        if e != nil {
                err = e
        }
        return err
}

// File creates a new Source for reading log entries from a file.
func File(name string, cfg *config.Config) (*Source, error) {
        var err error
        is := &Source{
                maxSize: cfg.MaxFileSizeBytes,
                name:    name,
        }

        is.file, err = os.Open(name)
        if err != nil {
                return nil, err
        }

        is.Seeker, err = os.Open(name)
        if err != nil {
                _ = is.file.Close()
                return nil, err
        }

        is.reader = bufio.NewReaderSize(io.LimitReader(is.file, is.maxSize), maxLineSize)
        return is, nil
}

// Reader creates a new Source for reading log entries from an io.Reader.  This will write the input to a temp file.
// which will be used to seek against.
func Reader(input io.Reader, cfg *config.Config) (*Source, error) {
        var err error
        is := &Source{
                maxSize: cfg.MaxFileSizeBytes,
        }

        // We will write the as read to a temp file.  Seek against the temp file.
        is.file, err = os.CreateTemp("", "jvl-*.log")
        if err != nil {
                return nil, err
        }

        // The io.TeeReader will write the input to the is.file as it is read.
        reader := io.TeeReader(input, is.file)

        // We can now seek against the data that is read in the input io.Reader.
        is.Seeker, err = os.Open(is.file.Name())
        if err != nil {
                _ = is.file.Close()
                return nil, err
        }

        reader = io.LimitReader(reader, is.maxSize)
        is.reader = bufio.NewReaderSize(reader, maxLineSize)
        return is, nil
}

func (is *Source) ParseLogEntries() (LazyLogEntries, error) {
        logEntries := make([]LazyLogEntry, 0, 1000)
        for {
                entry, err := is.ReadLogEntry()
                if err != nil {
                        if errors.Is(err, io.EOF) {
                                break
                        }

                        return LazyLogEntries{}, err
                }
                logEntries = append(logEntries, entry)
        }

        return LazyLogEntries{
                Seeker:  is.Seeker,
                Entries: logEntries,
        }, nil
}

func (is *Source) CanFollow() bool {
        return len(is.name) != 0
}

const ErrFileTruncated semerr.Error = "file truncated"

// ReadLogEntry reads the next ReadLogEntry from the file.
func (is *Source) ReadLogEntry() (LazyLogEntry, error) {
        for {
                if is.reader == nil {
                        // If we can't follow the file, or we have reached the max size, we are done.
                        if !is.CanFollow() || is.offset >= is.maxSize {
                                return LazyLogEntry{}, io.EOF
                        }

                        // has the file size changed since we last looked?
                        info, err := os.Stat(is.name)
                        if err != nil || is.prevFollowSize == info.Size() {
                                return LazyLogEntry{}, io.EOF
                        }

                        if info.Size() < is.offset {
                                // the file has been truncated or rolled over, all previous line offsets are invalid.
                                // we can't recover from this.
                                return LazyLogEntry{}, ErrFileTruncated
                        }
                        is.prevFollowSize = info.Size()
                        // reset the reader and try to read the file again.
                        _, _ = is.file.Seek(is.offset, io.SeekStart)
                        is.reader = bufio.NewReaderSize(io.LimitReader(is.file, is.maxSize-is.offset), maxLineSize)
                }

                line, err := is.reader.ReadSlice(byte('\n'))
                if err != nil {
                        if errors.Is(err, io.EOF) {
                                // set the reader to nil so that we can recover from EOF.
                                is.reader = nil
                        }
                        return LazyLogEntry{}, err
                }
                length := len(line)
                offset := is.offset
                is.offset += int64(length)
                if len(bytes.TrimSpace(line)) != 0 {
                        return LazyLogEntry{
                                offset: offset,
                                length: length,
                        }, nil
                }
        }
}

1	package source
2
3	import (
4	"bufio"
5	"bytes"
6	"errors"
7	"io"
8	"os"
9
10	"github.com/hedhyw/semerr/pkg/v1/semerr"
11
12	"github.com/hedhyw/json-log-viewer/internal/pkg/config"
13	)
14
15	const (
16	maxLineSize = 8 * 1024 * 1024
17	)
18
19	type Source struct {
20	// Seeker is used to do random access reads from the file.
21	Seeker *os.File
22	// Reader is used to read the file sequentially.
23	reader *bufio.Reader
24	// The log file we are reading from, or a temp file we are writing to (depending on if created with File or Reader func).
25	file *os.File
26	// offset is the next offset a long entry will be read from.
27	offset int64
28	// prevFollowSize is the size of the file the last time we checked
29	prevFollowSize int64
30	// name is the name of the file we are reading.
31	name string
32	// maxSize is the maximum size of the file we will read.
33	maxSize int64
34	}
35
36	func (is *Source) Close() (err error) {	38✔
37	err = is.file.Close()	38✔
38	e := is.Seeker.Close()	38✔
39	if e != nil {	38✔
NEW 40	err = e	×
NEW 41	}	×
42	return err	38✔
43	}
44
45	// File creates a new Source for reading log entries from a file.
46	func File(name string, cfg config.Config) (Source, error) {	30✔
47	var err error	30✔
48	is := &Source{	30✔
49	maxSize: cfg.MaxFileSizeBytes,	30✔
50	name: name,	30✔
51	}	30✔
52		30✔
53	is.file, err = os.Open(name)	30✔
54	if err != nil {	30✔
NEW 55	return nil, err	×
NEW 56	}	×
57
58	is.Seeker, err = os.Open(name)	30✔
59	if err != nil {	30✔
NEW 60	_ = is.file.Close()	×
NEW 61	return nil, err	×
NEW 62	}	×
63
64	is.reader = bufio.NewReaderSize(io.LimitReader(is.file, is.maxSize), maxLineSize)	30✔
65	return is, nil	30✔
66	}
67
68	// Reader creates a new Source for reading log entries from an io.Reader. This will write the input to a temp file.
69	// which will be used to seek against.
70	func Reader(input io.Reader, cfg config.Config) (Source, error) {	8✔
71	var err error	8✔
72	is := &Source{	8✔
73	maxSize: cfg.MaxFileSizeBytes,	8✔
74	}	8✔
75		8✔
76	// We will write the as read to a temp file. Seek against the temp file.	8✔
77	is.file, err = os.CreateTemp("", "jvl-*.log")	8✔
78	if err != nil {	8✔
NEW 79	return nil, err	×
NEW 80	}	×
81
82	// The io.TeeReader will write the input to the is.file as it is read.
83	reader := io.TeeReader(input, is.file)	8✔
84		8✔
85	// We can now seek against the data that is read in the input io.Reader.	8✔
86	is.Seeker, err = os.Open(is.file.Name())	8✔
87	if err != nil {	8✔
NEW 88	_ = is.file.Close()	×
NEW 89	return nil, err	×
NEW 90	}	×
91
92	reader = io.LimitReader(reader, is.maxSize)	8✔
93	is.reader = bufio.NewReaderSize(reader, maxLineSize)	8✔
94	return is, nil	8✔
95	}
96
97	func (is *Source) ParseLogEntries() (LazyLogEntries, error) {	38✔
98	logEntries := make([]LazyLogEntry, 0, 1000)	38✔
99	for {	1,128✔
100	entry, err := is.ReadLogEntry()	1,090✔
101	if err != nil {	1,128✔
102	if errors.Is(err, io.EOF) {	76✔
103	break	38✔
104	}
105
NEW 106	return LazyLogEntries{}, err	×
107	}
108	logEntries = append(logEntries, entry)	1,052✔
109	}
110
111	return LazyLogEntries{	38✔
112	Seeker: is.Seeker,	38✔
113	Entries: logEntries,	38✔
114	}, nil	38✔
115	}
116
NEW 117	func (is *Source) CanFollow() bool {	×
NEW 118	return len(is.name) != 0	×
NEW 119	}	×
120
121	const ErrFileTruncated semerr.Error = "file truncated"
122
123	// ReadLogEntry reads the next ReadLogEntry from the file.
124	func (is *Source) ReadLogEntry() (LazyLogEntry, error) {	1,090✔
125	for {	2,192✔
126	if is.reader == nil {	1,102✔
NEW 127	// If we can't follow the file, or we have reached the max size, we are done.	×
NEW 128	if !is.CanFollow() \|\| is.offset >= is.maxSize {	×
NEW 129	return LazyLogEntry{}, io.EOF	×
NEW 130	}	×
131
132	// has the file size changed since we last looked?
NEW 133	info, err := os.Stat(is.name)	×
NEW 134	if err != nil \|\| is.prevFollowSize == info.Size() {	×
NEW 135	return LazyLogEntry{}, io.EOF	×
NEW 136	}	×
137
NEW 138	if info.Size() < is.offset {	×
NEW 139	// the file has been truncated or rolled over, all previous line offsets are invalid.	×
NEW 140	// we can't recover from this.	×
NEW 141	return LazyLogEntry{}, ErrFileTruncated	×
NEW 142	}	×
NEW 143	is.prevFollowSize = info.Size()	×
NEW 144	// reset the reader and try to read the file again.	×
NEW 145	_, _ = is.file.Seek(is.offset, io.SeekStart)	×
NEW 146	is.reader = bufio.NewReaderSize(io.LimitReader(is.file, is.maxSize-is.offset), maxLineSize)	×
147	}
148
149	line, err := is.reader.ReadSlice(byte('\n'))	1,102✔
150	if err != nil {	1,140✔
151	if errors.Is(err, io.EOF) {	76✔
152	// set the reader to nil so that we can recover from EOF.	38✔
153	is.reader = nil	38✔
154	}	38✔
155	return LazyLogEntry{}, err	38✔
156	}
157	length := len(line)	1,064✔
158	offset := is.offset	1,064✔
159	is.offset += int64(length)	1,064✔
160	if len(bytes.TrimSpace(line)) != 0 {	2,116✔
161	return LazyLogEntry{	1,052✔
162	offset: offset,	1,052✔
163	length: length,	1,052✔
164	}, nil	1,052✔
165	}	1,052✔
166	}
167	}

hedhyw / json-log-viewer / 10831268990

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous