22280543736

Committed 22 Feb 2026 04:05PM UTC coverage: 90.845%. First build

Build # 22280543736

Build Type

Pull #11472

github

Committed by

brotherlogic

Commit Message

Fix: Reduce transient memory usage in GetLogs API

Optimized the loadAllLogs function to prevent excessive transient memory
allocation when handling GetLogs API requests. Previously, the function
would load all matching log entries into memory before applying the
limit of 20 logs.

This change introduces an early exit condition within the filepath.Walk
loops, ensuring that log processing stops as soon as 20 relevant log
entries are collected. This significantly reduces memory pressure and
improves performance, especially when dealing with large log files and
frequent GetLogs requests.

Closes #11471

Pull Request Pull Request #11472: Fix: Reduce transient memory usage in GetLogs API

Coverage Stats

2 of 6 new or added lines in 1 file covered. (33.33%)

129 of 142 relevant lines covered (90.85%)

1.1 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.0

/loggingutils.go

package main

import (
        "fmt"
        "io/ioutil"
        "os"
        "path/filepath"
        "sort"
        "strings"
        "time"

        "github.com/prometheus/client_golang/prometheus"
        "github.com/prometheus/client_golang/prometheus/promauto"
        "golang.org/x/net/context"
        "google.golang.org/protobuf/proto"

        pb "github.com/brotherlogic/logging/proto"
)

var (
        //DirSize - the print queue
        filtered = promauto.NewGauge(prometheus.GaugeOpts{
                Name: "logging_filtered",
                Help: "The size of the logs",
        })
        original = promauto.NewGauge(prometheus.GaugeOpts{
                Name: "logging_original",
                Help: "The size of the logs",
        })
        loaded = promauto.NewGauge(prometheus.GaugeOpts{
                Name: "logging_loaded",
                Help: "The size of the logs",
        })
)

func (s *Server) getFileName(origin string, timestamp int64) (string, string) {
        t := time.Unix(0, timestamp)
        return fmt.Sprintf("%v/%v/%v-%v-%v-%v.logs",
                        s.path,
                        origin,
                        t.Year(),
                        t.Month(),
                        t.Day(),
                        t.Hour()),
                fmt.Sprintf("%v/%v",
                        s.path,
                        origin)

}

func (s *Server) saveLogs(ctx context.Context, origin string, timestamp int64, logs []*pb.Log) error {
        fname, dir := s.getFileName(origin, timestamp)
        err := os.MkdirAll(dir, 0777)
        if err != nil {
                return err
        }

        data, err := s.marshal(logs)
        if err != nil {
                return err
        }

        return ioutil.WriteFile(fname, data, 0644)
}

func (s *Server) loadAllLogs(ctx context.Context, origin string, match string, includeDLogs bool, context string) ([]*pb.Log, error) {
        logs := []*pb.Log{}

        if !includeDLogs {
                err := filepath.Walk(s.path, func(path string, info os.FileInfo, err error) error {
                        if strings.Contains(path, origin) && !info.IsDir() {
                                nlogs, err := s.loadLogFile(path)
                                if err != nil {
                                        return err
                                }
                                for _, log := range nlogs {
                                        if match == "" || strings.Contains(log.GetLog(), match) {
                                                logs = append(logs, log)
                                                if len(logs) >= 20 {
                                                        return filepath.SkipDir
                                                }
                                        }
                                }
                        }
                        return nil
                })

                if err != nil {
                        return nil, err
                }
        }

        // Walk the dlogs if we've been asked to
        if includeDLogs {
                err := filepath.Walk(fmt.Sprintf("%v/%v", s.dpath, origin), func(path string, info os.FileInfo, err error) error {
                        if err == nil {
                                if (origin == "" || strings.Contains(path, origin)) && !info.IsDir() {
                                        dlogs, err := s.loadDLogFile(ctx, path, origin, context)

                                        if err != nil {
                                                return err
                                        }
                                        for _, log := range dlogs {
                                                if match == "" || strings.Contains(log.GetLog(), match) {
                                                        logs = append(logs, log)
                                                        if len(logs) >= 20 {
                                                                return filepath.SkipDir
                                                        }
                                                }
                                        }
                                }
                        }
                        return nil
                })

                if err != nil {
                        return nil, err
                }
        }

        sort.SliceStable(logs, func(i, j int) bool {
                return logs[i].GetTimestamp() > logs[j].GetTimestamp()
        })

        // Filter by context if need to
        var nlogs []*pb.Log
        for _, log := range logs {
                if context == "" || log.GetContext() == context {
                        nlogs = append(nlogs, log)
                }
        }

        original.Set(float64(len(logs)))
        filtered.Set(float64(len(nlogs)))

        // Only return 20 logs
        return nlogs[0:min(20, len(nlogs))], nil
}

func (s *Server) cleanAllLogs(ctx context.Context) error {
        var toDelete []string
        err := filepath.Walk(s.path, func(path string, info os.FileInfo, err error) error {
                if info != nil && !info.IsDir() {
                        s.DLog(ctx, fmt.Sprintf("Cleaning %v", path))
                        nlogs, err := s.loadLogFile(path)
                        if err != nil {
                                return err
                        }
                        newlogs := []*pb.Log{}
                        for _, log := range nlogs {
                                if time.Since(time.Unix(0, log.GetTimestamp())).Seconds() < float64(log.GetTtl()) && time.Since(time.Unix(0, log.GetTimestamp())).Seconds() > 0 {
                                        newlogs = append(newlogs, log)
                                }
                        }
                        if len(newlogs) > 0 {
                                data, err := s.marshal(newlogs)
                                if err == nil {
                                        err = ioutil.WriteFile(path, data, 0644)
                                        return err
                                }
                        } else {
                                // We can delete this file
                                toDelete = append(toDelete, path)

                        }
                }
                return nil
        })

        for _, td := range toDelete {
                os.Remove(td)
        }

        return err
}

func min(a, b int) int {
        if a < b {
                return a
        }
        return b
}

func (s *Server) loadLogs(ctx context.Context, origin string, timestamp int64) ([]*pb.Log, error) {
        fname, _ := s.getFileName(origin, timestamp)
        return s.loadLogFile(fname)
}

func (s *Server) loadLogFile(fname string) ([]*pb.Log, error) {
        if _, err := os.Stat(fname); os.IsNotExist(err) {
                return []*pb.Log{}, nil
        }

        data, err := s.load(fname)
        if err != nil {
                return nil, err
        }

        list := &pb.LogList{}
        proto.Unmarshal(data, list)
        return list.GetLogs(), nil
}

func (s *Server) loadDLogFile(ctx context.Context, fname, origin, context string) ([]*pb.Log, error) {
        return s.loadDLog(ctx, fname, origin, context)
}

1	package main
2
3	import (
4	"fmt"
5	"io/ioutil"
6	"os"
7	"path/filepath"
8	"sort"
9	"strings"
10	"time"
11
12	"github.com/prometheus/client_golang/prometheus"
13	"github.com/prometheus/client_golang/prometheus/promauto"
14	"golang.org/x/net/context"
15	"google.golang.org/protobuf/proto"
16
17	pb "github.com/brotherlogic/logging/proto"
18	)
19
20	var (
21	//DirSize - the print queue
22	filtered = promauto.NewGauge(prometheus.GaugeOpts{
23	Name: "logging_filtered",
24	Help: "The size of the logs",
25	})
26	original = promauto.NewGauge(prometheus.GaugeOpts{
27	Name: "logging_original",
28	Help: "The size of the logs",
29	})
30	loaded = promauto.NewGauge(prometheus.GaugeOpts{
31	Name: "logging_loaded",
32	Help: "The size of the logs",
33	})
34	)
35
36	func (s *Server) getFileName(origin string, timestamp int64) (string, string) {	1✔
37	t := time.Unix(0, timestamp)	1✔
38	return fmt.Sprintf("%v/%v/%v-%v-%v-%v.logs",	1✔
39	s.path,	1✔
40	origin,	1✔
41	t.Year(),	1✔
42	t.Month(),	1✔
43	t.Day(),	1✔
44	t.Hour()),	1✔
45	fmt.Sprintf("%v/%v",	1✔
46	s.path,	1✔
47	origin)	1✔
48		1✔
49	}	1✔
50
51	func (s Server) saveLogs(ctx context.Context, origin string, timestamp int64, logs []pb.Log) error {	1✔
52	fname, dir := s.getFileName(origin, timestamp)	1✔
53	err := os.MkdirAll(dir, 0777)	1✔
54	if err != nil {	1✔
55	return err	×
56	}	×
57
58	data, err := s.marshal(logs)	1✔
59	if err != nil {	2✔
60	return err	1✔
61	}	1✔
62
63	return ioutil.WriteFile(fname, data, 0644)	1✔
64	}
65
66	func (s Server) loadAllLogs(ctx context.Context, origin string, match string, includeDLogs bool, context string) ([]pb.Log, error) {	1✔
67	logs := []*pb.Log{}	1✔
68		1✔
69	if !includeDLogs {	2✔
70	err := filepath.Walk(s.path, func(path string, info os.FileInfo, err error) error {	2✔
71	if strings.Contains(path, origin) && !info.IsDir() {	2✔
72	nlogs, err := s.loadLogFile(path)	1✔
73	if err != nil {	2✔
74	return err	1✔
75	}	1✔
76	for _, log := range nlogs {	2✔
77	if match == "" \|\| strings.Contains(log.GetLog(), match) {	2✔
78	logs = append(logs, log)	1✔
79	if len(logs) >= 20 {	1✔
NEW 80	return filepath.SkipDir	×
NEW 81	}	×
82	}
83	}
84	}
85	return nil	1✔
86	})
87
88	if err != nil {	2✔
89	return nil, err	1✔
90	}	1✔
91	}
92
93	// Walk the dlogs if we've been asked to
94	if includeDLogs {	2✔
95	err := filepath.Walk(fmt.Sprintf("%v/%v", s.dpath, origin), func(path string, info os.FileInfo, err error) error {	2✔
96	if err == nil {	2✔
97	if (origin == "" \|\| strings.Contains(path, origin)) && !info.IsDir() {	2✔
98	dlogs, err := s.loadDLogFile(ctx, path, origin, context)	1✔
99		1✔
100	if err != nil {	2✔
101	return err	1✔
102	}	1✔
103	for _, log := range dlogs {	2✔
104	if match == "" \|\| strings.Contains(log.GetLog(), match) {	2✔
105	logs = append(logs, log)	1✔
106	if len(logs) >= 20 {	1✔
NEW 107	return filepath.SkipDir	×
NEW 108	}	×
109	}
110	}
111	}
112	}
113	return nil	1✔
114	})
115
116	if err != nil {	2✔
117	return nil, err	1✔
118	}	1✔
119	}
120
121	sort.SliceStable(logs, func(i, j int) bool {	2✔
122	return logs[i].GetTimestamp() > logs[j].GetTimestamp()	1✔
123	})	1✔
124
125	// Filter by context if need to
126	var nlogs []*pb.Log	1✔
127	for _, log := range logs {	2✔
128	if context == "" \|\| log.GetContext() == context {	2✔
129	nlogs = append(nlogs, log)	1✔
130	}	1✔
131	}
132
133	original.Set(float64(len(logs)))	1✔
134	filtered.Set(float64(len(nlogs)))	1✔
135		1✔
136	// Only return 20 logs	1✔
137	return nlogs[0:min(20, len(nlogs))], nil	1✔
138	}
139
140	func (s *Server) cleanAllLogs(ctx context.Context) error {	1✔
141	var toDelete []string	1✔
142	err := filepath.Walk(s.path, func(path string, info os.FileInfo, err error) error {	2✔
143	if info != nil && !info.IsDir() {	2✔
144	s.DLog(ctx, fmt.Sprintf("Cleaning %v", path))	1✔
145	nlogs, err := s.loadLogFile(path)	1✔
146	if err != nil {	2✔
147	return err	1✔
148	}	1✔
149	newlogs := []*pb.Log{}	1✔
150	for _, log := range nlogs {	2✔
151	if time.Since(time.Unix(0, log.GetTimestamp())).Seconds() < float64(log.GetTtl()) && time.Since(time.Unix(0, log.GetTimestamp())).Seconds() > 0 {	1✔
152	newlogs = append(newlogs, log)	×
153	}	×
154	}
155	if len(newlogs) > 0 {	1✔
156	data, err := s.marshal(newlogs)	×
157	if err == nil {	×
158	err = ioutil.WriteFile(path, data, 0644)	×
159	return err	×
160	}	×
161	} else {	1✔
162	// We can delete this file	1✔
163	toDelete = append(toDelete, path)	1✔
164		1✔
165	}	1✔
166	}
167	return nil	1✔
168	})
169
170	for _, td := range toDelete {	2✔
171	os.Remove(td)	1✔
172	}	1✔
173
174	return err	1✔
175	}
176
177	func min(a, b int) int {	1✔
178	if a < b {	2✔
179	return a	1✔
180	}	1✔
181	return b	1✔
182	}
183
184	func (s Server) loadLogs(ctx context.Context, origin string, timestamp int64) ([]pb.Log, error) {	1✔
185	fname, _ := s.getFileName(origin, timestamp)	1✔
186	return s.loadLogFile(fname)	1✔
187	}	1✔
188
189	func (s Server) loadLogFile(fname string) ([]pb.Log, error) {	1✔
190	if _, err := os.Stat(fname); os.IsNotExist(err) {	2✔
191	return []*pb.Log{}, nil	1✔
192	}	1✔
193
194	data, err := s.load(fname)	1✔
195	if err != nil {	2✔
196	return nil, err	1✔
197	}	1✔
198
199	list := &pb.LogList{}	1✔
200	proto.Unmarshal(data, list)	1✔
201	return list.GetLogs(), nil	1✔
202	}
203
204	func (s Server) loadDLogFile(ctx context.Context, fname, origin, context string) ([]pb.Log, error) {	1✔
205	return s.loadDLog(ctx, fname, origin, context)	1✔
206	}	1✔

brotherlogic / logging / 22280543736

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous