couchbase / moss / 190

pending completion

Build # 190

Build Type

push

travis-ci

Committed by

abhinavdangeti

Commit Message

<a href="https://github.com/couchbase/moss/commit/<a class=hub.com/couchbase/moss/commit/615556427b382f9c5f43937a7192152096060c53">615556427<a href="https://github.com/couchbase/moss/commit/615556427b382f9c5f43937a7192152096060c53">&quot;&gt;Update --help for mossScope import, minor fixes

+ The import feature accepts as input a JSON array of maps,
  where the mapping is a string to string only for now.
+ When a value of &amp;quot;k&amp;quot; is not included in the input JSON, do
  not write the key-value into moss store.
+ Correctly account for the number of key-values added to
  the store via import.

Change-Id: I</a><a class="double-link" href="https://github.com/couchbase/moss/commit/<a class="double-link" href="https://github.com/couchbase/moss/commit/6464b5c723f28934d4734c657e26010366bfb447">6464b5c72</a>">6464b5c72</a><a href="https://github.com/couchbase/moss/commit/615556427b382f9c5f43937a7192152096060c53">
Reviewed-on: http://review.couchbase.org/74060
Reviewed-by: Steve Yen &lt;steve.yen@gmail.com>
Tested-by: abhinav dangeti <abhinav@couchbase.com>

Run Details

21 of 21 new or added lines in 2 files covered. (100.0%)

2865 of 3581 relevant lines covered (80.01%)

8839.67 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

59.81

/cmd/mossScope/cmd/import.go

// Copyright © 2017 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
        "bufio"
        "encoding/json"
        "fmt"
        "io/ioutil"
        "math"
        "os"
        "sync"

        "github.com/couchbase/moss"
        "github.com/spf13/cobra"
)

// importCmd represents the import command
var importCmd = &cobra.Command{
        Use:   "import",
        Short: "Imports the docs from the JSON file into the store",
        Long: `Imports the key-values from the specified file (required to be in
JSON format - array of maps - mapping a string to string only),
taking into account - batch size, which can be specified by an
optional flag, into the store. For example:
        ./mossScope import <path_to_store> <flag(s)>
Order of execution (if all flags included): stdin < cmdline < file
Expected JSON file format:
        [{"k" : "key0", "v" : "val0"}, {"k" : "key1", "v" : "val1"}]`,

        PreRunE: func(cmd *cobra.Command, args []string) error {
                if len(args) < 1 {
                        return fmt.Errorf("One path needed!")
                } else if len(args) != 1 {
                        return fmt.Errorf("Only one path allowed!")
                }

                if len(fileInput) == 0 && len(jsonInput) == 0 && !readFromStdin {
                        return fmt.Errorf("At least one input source required!")
                }

                return nil
        },

        RunE: func(cmd *cobra.Command, args []string) error {
                from_file := ""
                from_cla := ""
                from_stdin := ""

                var err error

                if len(fileInput) > 0 {
                        input, err := ioutil.ReadFile(fileInput)
                        if err != nil {
                                return fmt.Errorf("File read error: %v", err)
                        }
                        from_file = string(input)
                }

                if len(jsonInput) > 0 {
                        from_cla = jsonInput
                }

                if readFromStdin {
                        reader := bufio.NewReader(os.Stdin)
                        from_stdin, err = reader.ReadString('\n')
                        if err != nil {
                                return fmt.Errorf("Error in reading from stdin, err: %v", err)
                        }
                }

                err = invokeImport(from_stdin, args[0])
                if err != nil {
                        return fmt.Errorf("Import from STDIN failed; err: %v", err)
                }

                err = invokeImport(from_cla, args[0])
                if err != nil {
                        return fmt.Errorf("Import from CMD-LINE failed; err: %v", err)
                }

                err = invokeImport(from_file, args[0])
                if err != nil {
                        return fmt.Errorf("Import from FILE failed; err: %v", err)
                }

                return nil
        },
}

var batchSize int
var fileInput string
var jsonInput string
var readFromStdin bool

type KV struct {
        Key string `json:"k"`
        Val string `json:"v"`
}

func invokeImport(jsonStr string, dir string) error {
        if len(jsonStr) == 0 {
                return nil
        }

        input := []byte(jsonStr)

        var data []KV
        err := json.Unmarshal(input, &data)
        if err != nil {
                fmt.Printf("Expected format:")
                fmt.Printf("[{\"k\" : \"key0\", \"v\" : \"val0\"}, " +
                        "{\"k\" : \"key1\", \"v\" : \"val1\"}]\n")
                return fmt.Errorf("Json-UnMarshal() failed!, err: %v", err)
        }

        if len(data) == 0 {
                fmt.Println("Empty JSON file, no key-values to load!")
                return nil
        }

        if _, err := os.Stat(dir); os.IsNotExist(err) {
                // Create the directory (specified) if it does not already exist
                os.Mkdir(dir, 0777)
        }

        var m sync.Mutex
        var waitingForCleanCh chan struct{}

        var store *moss.Store
        var coll moss.Collection

        co := moss.CollectionOptions{
                OnEvent: func(event moss.Event) {
                        if event.Kind == moss.EventKindPersisterProgress {
                                stats, err := coll.Stats()
                                if err == nil && stats.CurDirtyOps <= 0 &&
                                        stats.CurDirtyBytes <= 0 && stats.CurDirtySegments <= 0 {
                                        m.Lock()
                                        if waitingForCleanCh != nil {
                                                waitingForCleanCh <- struct{}{}
                                                waitingForCleanCh = nil
                                        }
                                        m.Unlock()
                                }
                        }
                },
        }

        store, coll, err = moss.OpenStoreCollection(dir,
                moss.StoreOptions{CollectionOptions: co},
                moss.StorePersistOptions{})
        if err != nil || store == nil {
                return fmt.Errorf("Moss-OpenStoreCollection failed, err: %v", err)
        }

        defer store.Close()
        defer coll.Close()

        ch := make(chan struct{}, 1)

        numBatches := 1
        itemsWritten := 0

        if batchSize <= 0 {
                // All key-values in a single batch

                sizeOfBatch := 0
                for i := 0; i < len(data); i++ {
                        // Get the size of the batch
                        sizeOfBatch += len(data[i].Key) + len(data[i].Val)
                }

                if sizeOfBatch == 0 {
                        return nil
                }

                batch, err := coll.NewBatch(len(data), sizeOfBatch)
                if err != nil {
                        return fmt.Errorf("Collection-NewBatch() failed, err: %v", err)
                }

                for i := 0; i < len(data); i++ {
                        if len(data[i].Key) == 0 {
                                continue
                        }

                        kbuf, err := batch.Alloc(len(data[i].Key))
                        if err != nil {
                                return fmt.Errorf("Batch-Alloc() failed, err: %v", err)
                        }
                        vbuf, err := batch.Alloc(len(data[i].Val))
                        if err != nil {
                                return fmt.Errorf("Batch-Alloc() failed, err: %v", err)
                        }

                        copy(kbuf, data[i].Key)
                        copy(vbuf, data[i].Val)

                        err = batch.AllocSet(kbuf, vbuf)
                        if err != nil {
                                return fmt.Errorf("Batch-AllocSet() failed, err: %v", err)
                        }
                        itemsWritten++
                }

                m.Lock()
                waitingForCleanCh = ch
                m.Unlock()

                err = coll.ExecuteBatch(batch, moss.WriteOptions{})
                if err != nil {
                        return fmt.Errorf("Collection-ExecuteBatch() failed, err: %v", err)
                }

        } else {

                numBatches = int(math.Ceil(float64(len(data)) / float64(batchSize)))
                cursor := 0

                for i := 0; i < numBatches; i++ {
                        sizeOfBatch := 0
                        numItemsInBatch := 0
                        for j := cursor; j < cursor+batchSize; j++ {
                                if j >= len(data) {
                                        break
                                }
                                sizeOfBatch += len(data[j].Key) + len(data[j].Val)
                                numItemsInBatch++
                        }
                        if sizeOfBatch == 0 {
                                continue
                        }

                        batch, err := coll.NewBatch(numItemsInBatch, sizeOfBatch)
                        if err != nil {
                                return fmt.Errorf("Collection-NewBatch() failed, err: %v", err)
                        }

                        for j := 0; j < numItemsInBatch; j++ {
                                if len(data[cursor].Key) == 0 {
                                        cursor++
                                        continue
                                }

                                kbuf, err := batch.Alloc(len(data[cursor].Key))
                                if err != nil {
                                        return fmt.Errorf("Batch-Alloc() failed, err: %v", err)
                                }
                                vbuf, err := batch.Alloc(len(data[cursor].Val))
                                if err != nil {
                                        return fmt.Errorf("Batch-Alloc() failed, err: %v", err)
                                }

                                copy(kbuf, data[cursor].Key)
                                copy(vbuf, data[cursor].Val)

                                err = batch.AllocSet(kbuf, vbuf)
                                if err != nil {
                                        return fmt.Errorf("Batch-AllocSet() failed, err: %v", err)
                                }
                                cursor++
                                itemsWritten++
                        }

                        m.Lock()
                        waitingForCleanCh = ch
                        m.Unlock()

                        err = coll.ExecuteBatch(batch, moss.WriteOptions{})
                        if err != nil {
                                return fmt.Errorf("Collection-ExecuteBatch() failed, err: %v",
                                        err)
                        }
                }
        }

        <-ch

        fmt.Printf("DONE! .. Wrote %d key-values, in %d batch(es)\n",
                itemsWritten, numBatches)

        return nil
}

func init() {
        RootCmd.AddCommand(importCmd)

        // Local flags that is intended to work as a flag over import
        importCmd.Flags().IntVar(&batchSize, "batchsize", 0,
                "Batch-size for the set operations (default: all docs in one batch)")
        importCmd.Flags().StringVar(&fileInput, "file", "",
                "Reads JSON content from file")
        importCmd.Flags().StringVar(&jsonInput, "json", "",
                "Reads JSON content from command-line")
        importCmd.Flags().BoolVar(&readFromStdin, "stdin", false,
                "Reads JSON content from stdin (Enter to submit)")
}

1	// Copyright © 2017 Couchbase, Inc.
2	//
3	// Licensed under the Apache License, Version 2.0 (the "License");
4	// you may not use this file except in compliance with the License.
5	// You may obtain a copy of the License at
6	//
7	// http://www.apache.org/licenses/LICENSE-2.0
8	//
9	// Unless required by applicable law or agreed to in writing, software
10	// distributed under the License is distributed on an "AS IS" BASIS,
11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	// See the License for the specific language governing permissions and
13	// limitations under the License.
14
15	package cmd
16
17	import (
18	"bufio"
19	"encoding/json"
20	"fmt"
21	"io/ioutil"
22	"math"
23	"os"
24	"sync"
25
26	"github.com/couchbase/moss"
27	"github.com/spf13/cobra"
28	)
29
30	// importCmd represents the import command
31	var importCmd = &cobra.Command{
32	Use: "import",
33	Short: "Imports the docs from the JSON file into the store",
34	Long: `Imports the key-values from the specified file (required to be in
35	JSON format - array of maps - mapping a string to string only),
36	taking into account - batch size, which can be specified by an
37	optional flag, into the store. For example:
38	./mossScope import <path_to_store> <flag(s)>
39	Order of execution (if all flags included): stdin < cmdline < file
40	Expected JSON file format:
41	[{"k" : "key0", "v" : "val0"}, {"k" : "key1", "v" : "val1"}]`,
42
43	PreRunE: func(cmd *cobra.Command, args []string) error {	×
44	if len(args) < 1 {	×
45	return fmt.Errorf("One path needed!")	×
46	} else if len(args) != 1 {	×
47	return fmt.Errorf("Only one path allowed!")	×
48	}	×
49
50	if len(fileInput) == 0 && len(jsonInput) == 0 && !readFromStdin {	×
51	return fmt.Errorf("At least one input source required!")	×
52	}	×
53
54	return nil	×
55	},
56
57	RunE: func(cmd *cobra.Command, args []string) error {	×
58	from_file := ""	×
59	from_cla := ""	×
60	from_stdin := ""	×
61		×
62	var err error	×
63		×
64	if len(fileInput) > 0 {	×
65	input, err := ioutil.ReadFile(fileInput)	×
66	if err != nil {	×
67	return fmt.Errorf("File read error: %v", err)	×
68	}	×
69	from_file = string(input)	×
70	}
71
72	if len(jsonInput) > 0 {	×
73	from_cla = jsonInput	×
74	}	×
75
76	if readFromStdin {	×
77	reader := bufio.NewReader(os.Stdin)	×
78	from_stdin, err = reader.ReadString('\n')	×
79	if err != nil {	×
80	return fmt.Errorf("Error in reading from stdin, err: %v", err)	×
81	}	×
82	}
83
84	err = invokeImport(from_stdin, args[0])	×
85	if err != nil {	×
86	return fmt.Errorf("Import from STDIN failed; err: %v", err)	×
87	}	×
88
89	err = invokeImport(from_cla, args[0])	×
90	if err != nil {	×
91	return fmt.Errorf("Import from CMD-LINE failed; err: %v", err)	×
92	}	×
93
94	err = invokeImport(from_file, args[0])	×
95	if err != nil {	×
96	return fmt.Errorf("Import from FILE failed; err: %v", err)	×
97	}	×
98
99	return nil	×
100	},
101	}
102
103	var batchSize int
104	var fileInput string
105	var jsonInput string
106	var readFromStdin bool
107
108	type KV struct {
109	Key string `json:"k"`
110	Val string `json:"v"`
111	}
112
113	func invokeImport(jsonStr string, dir string) error {	2✔
114	if len(jsonStr) == 0 {	2✔
115	return nil	×
116	}	×
117
118	input := []byte(jsonStr)	2✔
119		2✔
120	var data []KV	2✔
121	err := json.Unmarshal(input, &data)	2✔
122	if err != nil {	2✔
123	fmt.Printf("Expected format:")	×
124	fmt.Printf("[{\"k\" : \"key0\", \"v\" : \"val0\"}, " +	×
125	"{\"k\" : \"key1\", \"v\" : \"val1\"}]\n")	×
126	return fmt.Errorf("Json-UnMarshal() failed!, err: %v", err)	×
127	}	×
128
129	if len(data) == 0 {	2✔
130	fmt.Println("Empty JSON file, no key-values to load!")	×
131	return nil	×
132	}	×
133
134	if _, err := os.Stat(dir); os.IsNotExist(err) {	4✔
135	// Create the directory (specified) if it does not already exist	2✔
136	os.Mkdir(dir, 0777)	2✔
137	}	2✔
138
139	var m sync.Mutex	2✔
140	var waitingForCleanCh chan struct{}	2✔
141		2✔
142	var store *moss.Store	2✔
143	var coll moss.Collection	2✔
144		2✔
145	co := moss.CollectionOptions{	2✔
146	OnEvent: func(event moss.Event) {	24✔
147	if event.Kind == moss.EventKindPersisterProgress {	25✔
148	stats, err := coll.Stats()	3✔
149	if err == nil && stats.CurDirtyOps <= 0 &&	3✔
150	stats.CurDirtyBytes <= 0 && stats.CurDirtySegments <= 0 {	5✔
151	m.Lock()	2✔
152	if waitingForCleanCh != nil {	4✔
153	waitingForCleanCh <- struct{}{}	2✔
154	waitingForCleanCh = nil	2✔
155	}	2✔
156	m.Unlock()	2✔
157	}
158	}
159	},
160	}
161
162	store, coll, err = moss.OpenStoreCollection(dir,	2✔
163	moss.StoreOptions{CollectionOptions: co},	2✔
164	moss.StorePersistOptions{})	2✔
165	if err != nil \|\| store == nil {	2✔
166	return fmt.Errorf("Moss-OpenStoreCollection failed, err: %v", err)	×
167	}	×
168
169	defer store.Close()	2✔
170	defer coll.Close()	2✔
171		2✔
172	ch := make(chan struct{}, 1)	2✔
173		2✔
174	numBatches := 1	2✔
175	itemsWritten := 0	2✔
176		2✔
177	if batchSize <= 0 {	3✔
178	// All key-values in a single batch	1✔
179		1✔
180	sizeOfBatch := 0	1✔
181	for i := 0; i < len(data); i++ {	11✔
182	// Get the size of the batch	10✔
183	sizeOfBatch += len(data[i].Key) + len(data[i].Val)	10✔
184	}	10✔
185
186	if sizeOfBatch == 0 {	1✔
187	return nil	×
188	}	×
189
190	batch, err := coll.NewBatch(len(data), sizeOfBatch)	1✔
191	if err != nil {	1✔
192	return fmt.Errorf("Collection-NewBatch() failed, err: %v", err)	×
193	}	×
194
195	for i := 0; i < len(data); i++ {	11✔
196	if len(data[i].Key) == 0 {	10✔
197	continue	×
198	}
199
200	kbuf, err := batch.Alloc(len(data[i].Key))	10✔
201	if err != nil {	10✔
202	return fmt.Errorf("Batch-Alloc() failed, err: %v", err)	×
203	}	×
204	vbuf, err := batch.Alloc(len(data[i].Val))	10✔
205	if err != nil {	10✔
206	return fmt.Errorf("Batch-Alloc() failed, err: %v", err)	×
207	}	×
208
209	copy(kbuf, data[i].Key)	10✔
210	copy(vbuf, data[i].Val)	10✔
211		10✔
212	err = batch.AllocSet(kbuf, vbuf)	10✔
213	if err != nil {	10✔
214	return fmt.Errorf("Batch-AllocSet() failed, err: %v", err)	×
215	}	×
216	itemsWritten++	10✔
217	}
218
219	m.Lock()	1✔
220	waitingForCleanCh = ch	1✔
221	m.Unlock()	1✔
222		1✔
223	err = coll.ExecuteBatch(batch, moss.WriteOptions{})	1✔
224	if err != nil {	1✔
225	return fmt.Errorf("Collection-ExecuteBatch() failed, err: %v", err)	×
226	}	×
227
228	} else {	1✔
229		1✔
230	numBatches = int(math.Ceil(float64(len(data)) / float64(batchSize)))	1✔
231	cursor := 0	1✔
232		1✔
233	for i := 0; i < numBatches; i++ {	5✔
234	sizeOfBatch := 0	4✔
235	numItemsInBatch := 0	4✔
236	for j := cursor; j < cursor+batchSize; j++ {	15✔
237	if j >= len(data) {	12✔
238	break	1✔
239	}
240	sizeOfBatch += len(data[j].Key) + len(data[j].Val)	10✔
241	numItemsInBatch++	10✔
242	}
243	if sizeOfBatch == 0 {	4✔
244	continue	×
245	}
246
247	batch, err := coll.NewBatch(numItemsInBatch, sizeOfBatch)	4✔
248	if err != nil {	4✔
249	return fmt.Errorf("Collection-NewBatch() failed, err: %v", err)	×
250	}	×
251
252	for j := 0; j < numItemsInBatch; j++ {	14✔
253	if len(data[cursor].Key) == 0 {	10✔
254	cursor++	×
255	continue	×
256	}
257
258	kbuf, err := batch.Alloc(len(data[cursor].Key))	10✔
259	if err != nil {	10✔
260	return fmt.Errorf("Batch-Alloc() failed, err: %v", err)	×
261	}	×
262	vbuf, err := batch.Alloc(len(data[cursor].Val))	10✔
263	if err != nil {	10✔
264	return fmt.Errorf("Batch-Alloc() failed, err: %v", err)	×
265	}	×
266
267	copy(kbuf, data[cursor].Key)	10✔
268	copy(vbuf, data[cursor].Val)	10✔
269		10✔
270	err = batch.AllocSet(kbuf, vbuf)	10✔
271	if err != nil {	10✔
272	return fmt.Errorf("Batch-AllocSet() failed, err: %v", err)	×
273	}	×
274	cursor++	10✔
275	itemsWritten++	10✔
276	}
277
278	m.Lock()	4✔
279	waitingForCleanCh = ch	4✔
280	m.Unlock()	4✔
281		4✔
282	err = coll.ExecuteBatch(batch, moss.WriteOptions{})	4✔
283	if err != nil {	4✔
284	return fmt.Errorf("Collection-ExecuteBatch() failed, err: %v",	×
285	err)	×
286	}	×
287	}
288	}
289
290	<-ch	2✔
291		2✔
292	fmt.Printf("DONE! .. Wrote %d key-values, in %d batch(es)\n",	2✔
293	itemsWritten, numBatches)	2✔
294		2✔
295	return nil	2✔
296	}
297
298	func init() {	2✔
299	RootCmd.AddCommand(importCmd)	2✔
300		2✔
301	// Local flags that is intended to work as a flag over import	2✔
302	importCmd.Flags().IntVar(&batchSize, "batchsize", 0,	2✔
303	"Batch-size for the set operations (default: all docs in one batch)")	2✔
304	importCmd.Flags().StringVar(&fileInput, "file", "",	2✔
305	"Reads JSON content from file")	2✔
306	importCmd.Flags().StringVar(&jsonInput, "json", "",	2✔
307	"Reads JSON content from command-line")	2✔
308	importCmd.Flags().BoolVar(&readFromStdin, "stdin", false,	2✔
309	"Reads JSON content from stdin (Enter to submit)")	2✔
310	}	2✔

couchbase / moss / 190

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous