couchbase / moss / 173

pending completion

Build # 173

Build Type

push

travis-ci

Committed by

abhinavdangeti

Commit Message

MB-22540: Add a "diag" option for mossScope stats

+ cbCollectInfo will invoke stats with this option
+ More stats will be added to the diag option of mossScope:stats

Change-Id: I460d05110
Reviewed-on: http://review.couchbase.org/72975
Reviewed-by: Steve Yen <steve.yen@gmail.com>
Tested-by: abhinav dangeti <abhinav@couchbase.com>

Run Details

141 of 141 new or added lines in 4 files covered. (100.0%)

2700 of 3385 relevant lines covered (79.76%)

8393.72 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

71.15

/cmd/mossScope/cmd/fragStats.go

// Copyright © 2017 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
        "encoding/json"
        "fmt"

        "github.com/couchbase/moss"
        "github.com/spf13/cobra"
)

// fragStatsCmd represents the frag command
var fragStatsCmd = &cobra.Command{
        Use:   "fragmentation",
        Short: "Dumps the fragmentation stats",
        Long: `This command dumps the key-val sizes and directory size info,
and alongside that estimates the fragmentation levels. This data
could assist with decisions around invoking manual compaction.
        ./mossScope stats frag <path_to_store>`,

        PreRunE: func(cmd *cobra.Command, args []string) error {
                if len(args) < 1 {
                        return fmt.Errorf("At least one path is required!")
                }
                return nil
        },

        RunE: func(cmd *cobra.Command, args []string) error {
                return invokeFragStats(args)
        },
}

func invokeFragStats(dirs []string) error {
        if jsonFormat {
                fmt.Printf("[")
        }
        for index, dir := range dirs {
                store, err := moss.OpenStore(dir, moss.StoreOptions{})
                if err != nil || store == nil {
                        return fmt.Errorf("Moss-OpenStore() API failed, err: %v", err)
                }
                defer store.Close()

                stats_map := make(map[string]int64)

                err = fetchFragStats(store, stats_map)
                if err != nil {
                        return err
                }

                if jsonFormat {
                        jBuf, err := json.Marshal(stats_map)
                        if err != nil {
                                return fmt.Errorf("Json-Marshal() failed!, err: %v", err)
                        }
                        if index != 0 {
                                fmt.Printf(",")
                        }
                        fmt.Printf("{\"%s\":", dir)
                        fmt.Printf("%s}", string(jBuf))
                } else {
                        fmt.Println(dir)
                        for k, v := range stats_map {
                                fmt.Printf("%25s : %v\n", k, v)
                        }
                        fmt.Println()
                }
        }

        if jsonFormat {
                fmt.Printf("]\n")
        }

        return nil
}

func fetchFragStats(store *moss.Store, stats map[string]int64) error {
        if store == nil {
                return nil
        }

        stats["data_bytes"] = 0
        stats["dir_size"] = 0
        stats["fragmentation_bytes"] = 0
        stats["fragmentation_percent"] = 0

        curr_snap, err := store.Snapshot()
        if err != nil || curr_snap == nil {
                return nil
        }

        footer := curr_snap.(*moss.Footer)

        // Acquire key, val bytes from all segments of latest footer
        for i := range footer.SegmentLocs {
                sloc := &footer.SegmentLocs[i]

                stats["data_bytes"] += int64(sloc.TotKeyByte)
                stats["data_bytes"] += int64(sloc.TotValByte)
        }

        for {
                // header signature
                stats["data_bytes"] += 4096

                // footer signature
                footer = curr_snap.(*moss.Footer)
                jBuf, err := json.Marshal(footer)
                if err != nil {
                        return fmt.Errorf("Json-Marshal() failed!, err: %v", err)
                }

                stats["data_bytes"] += int64(len(jBuf)) // footer length

                // Also account for the magic that repeats twice at start and end
                stats["data_bytes"] += int64(2 * len(moss.STORE_MAGIC_BEG))
                stats["data_bytes"] += int64(2 * len(moss.STORE_MAGIC_END))

                prev_snap, err := store.SnapshotPrevious(curr_snap)
                curr_snap.Close()
                curr_snap = prev_snap

                if err != nil || curr_snap == nil {
                        break
                }
        }

        sstats, err := store.Stats()
        if err != nil {
                return fmt.Errorf("Store-Stats() failed!, err: %v", err)
        }

        stats["dir_size"] = int64(sstats["num_bytes_used_disk"].(uint64))

        stats["fragmentation_bytes"] = stats["dir_size"] - stats["data_bytes"]
        stats["fragmentation_percent"] = int64(100 *
                ((float64(stats["fragmentation_bytes"])) /
                        float64(stats["dir_size"])))

        return nil
}

func init() {
        statsCmd.AddCommand(fragStatsCmd)
}

1	// Copyright © 2017 Couchbase, Inc.
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// http://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14
15	package cmd
16
17	import (
18	"encoding/json"
19	"fmt"
20
21	"github.com/couchbase/moss"
22	"github.com/spf13/cobra"
23	)
24
25	// fragStatsCmd represents the frag command
26	var fragStatsCmd = &cobra.Command{
27	Use: "fragmentation",
28	Short: "Dumps the fragmentation stats",
29	Long: `This command dumps the key-val sizes and directory size info,
30	and alongside that estimates the fragmentation levels. This data
31	could assist with decisions around invoking manual compaction.
32	./mossScope stats frag <path_to_store>`,
33
34	PreRunE: func(cmd *cobra.Command, args []string) error {	×
35	if len(args) < 1 {	×
36	return fmt.Errorf("At least one path is required!")	×
37	}	×
38	return nil	×
39	},
40
41	RunE: func(cmd *cobra.Command, args []string) error {	×
42	return invokeFragStats(args)	×
43	},	×
44	}
45
46	func invokeFragStats(dirs []string) error {	1✔
47	if jsonFormat {	2✔
48	fmt.Printf("[")	1✔
49	}	1✔
50	for index, dir := range dirs {	2✔
51	store, err := moss.OpenStore(dir, moss.StoreOptions{})	1✔
52	if err != nil \|\| store == nil {	1✔
53	return fmt.Errorf("Moss-OpenStore() API failed, err: %v", err)	×
54	}	×
55	defer store.Close()	1✔
56		1✔
57	stats_map := make(map[string]int64)	1✔
58		1✔
59	err = fetchFragStats(store, stats_map)	1✔
60	if err != nil {	1✔
61	return err	×
62	}	×
63
64	if jsonFormat {	2✔
65	jBuf, err := json.Marshal(stats_map)	1✔
66	if err != nil {	1✔
67	return fmt.Errorf("Json-Marshal() failed!, err: %v", err)	×
68	}	×
69	if index != 0 {	1✔
70	fmt.Printf(",")	×
71	}	×
72	fmt.Printf("{\"%s\":", dir)	1✔
73	fmt.Printf("%s}", string(jBuf))	1✔
74	} else {	×
75	fmt.Println(dir)	×
76	for k, v := range stats_map {	×
77	fmt.Printf("%25s : %v\n", k, v)	×
78	}	×
79	fmt.Println()	×
80	}
81	}
82
83	if jsonFormat {	2✔
84	fmt.Printf("]\n")	1✔
85	}	1✔
86
87	return nil	1✔
88	}
89
90	func fetchFragStats(store *moss.Store, stats map[string]int64) error {	1✔
91	if store == nil {	1✔
92	return nil	×
93	}	×
94
95	stats["data_bytes"] = 0	1✔
96	stats["dir_size"] = 0	1✔
97	stats["fragmentation_bytes"] = 0	1✔
98	stats["fragmentation_percent"] = 0	1✔
99		1✔
100	curr_snap, err := store.Snapshot()	1✔
101	if err != nil \|\| curr_snap == nil {	1✔
102	return nil	×
103	}	×
104
105	footer := curr_snap.(*moss.Footer)	1✔
106		1✔
107	// Acquire key, val bytes from all segments of latest footer	1✔
108	for i := range footer.SegmentLocs {	3✔
109	sloc := &footer.SegmentLocs[i]	2✔
110		2✔
111	stats["data_bytes"] += int64(sloc.TotKeyByte)	2✔
112	stats["data_bytes"] += int64(sloc.TotValByte)	2✔
113	}	2✔
114
115	for {	3✔
116	// header signature	2✔
117	stats["data_bytes"] += 4096	2✔
118		2✔
119	// footer signature	2✔
120	footer = curr_snap.(*moss.Footer)	2✔
121	jBuf, err := json.Marshal(footer)	2✔
122	if err != nil {	2✔
123	return fmt.Errorf("Json-Marshal() failed!, err: %v", err)	×
124	}	×
125
126	stats["data_bytes"] += int64(len(jBuf)) // footer length	2✔
127		2✔
128	// Also account for the magic that repeats twice at start and end	2✔
129	stats["data_bytes"] += int64(2 * len(moss.STORE_MAGIC_BEG))	2✔
130	stats["data_bytes"] += int64(2 * len(moss.STORE_MAGIC_END))	2✔
131		2✔
132	prev_snap, err := store.SnapshotPrevious(curr_snap)	2✔
133	curr_snap.Close()	2✔
134	curr_snap = prev_snap	2✔
135		2✔
136	if err != nil \|\| curr_snap == nil {	3✔
137	break	1✔
138	}
139	}
140
141	sstats, err := store.Stats()	1✔
142	if err != nil {	1✔
143	return fmt.Errorf("Store-Stats() failed!, err: %v", err)	×
144	}	×
145
146	stats["dir_size"] = int64(sstats["num_bytes_used_disk"].(uint64))	1✔
147		1✔
148	stats["fragmentation_bytes"] = stats["dir_size"] - stats["data_bytes"]	1✔
149	stats["fragmentation_percent"] = int64(100 *	1✔
150	((float64(stats["fragmentation_bytes"])) /	1✔
151	float64(stats["dir_size"])))	1✔
152		1✔
153	return nil	1✔
154	}
155
156	func init() {	2✔
157	statsCmd.AddCommand(fragStatsCmd)	2✔
158	}	2✔

couchbase / moss / 173

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous