couchbase / moss / 194

pending completion

Build # 194

Build Type

push

travis-ci

Committed by

hisundar

Commit Message

<a href="https://github.com/couchbase/moss/commit/<a class=hub.com/couchbase/moss/commit/c63a3aff41424cbed896f8b5a9b50e90910494b9">c63a3aff4<a href="https://github.com/couchbase/moss/commit/c63a3aff41424cbed896f8b5a9b50e90910494b9">&quot;&gt;Fix sporadic &amp;quot;loadSegments mmap.Map(), err: invalid argument&amp;quot;

With child collections some merge segments can be empty.
Happens when multiple go routines exclusively load to only their
child collection.
Avoid persisting empty merge segments to prevent mmap failures.
Only append non-empty batches in ExecuteBatch()
Use correct len in isEmpty()
Use storeFooter in buildNewFooter()

Change-Id: I</a><a class="double-link" href="https://github.com/couchbase/moss/commit/<a class="double-link" href="https://github.com/couchbase/moss/commit/f5e6b76ccde9cfb07a22387b3e2edba1b97f3590">f5e6b76cc</a>">f5e6b76cc</a><a href="https://github.com/couchbase/moss/commit/c63a3aff41424cbed896f8b5a9b50e90910494b9">
Reviewed-on: http://review.couchbase.org/74491
Reviewed-by: Steve Yen &lt;steve.yen@gmail.com>
Tested-by: Sundararaman Sridharan <sundar@couchbase.com>

Run Details

15 of 15 new or added lines in 4 files covered. (100.0%)

3162 of 4005 relevant lines covered (78.95%)

10286.77 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

70.21

/store_footer.go

//  Copyright (c) 2016 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package moss

import (
        "bytes"
        "encoding/binary"
        "encoding/json"
        "fmt"
        "time"

        "github.com/edsrzf/mmap-go"
)

func (s *Store) persistFooter(file File, footer *Footer,
        options StorePersistOptions) error {
        startTime := time.Now()
        if !options.NoSync {
                err := file.Sync()
                if err != nil {
                        return err
                }
        }

        err := s.persistFooterUnsynced(file, footer)
        if err != nil {
                return err
        }

        if !options.NoSync {
                err = file.Sync()
        }

        if err == nil {
                s.histograms["PersistFooterUsecs"].Add(
                        uint64(time.Since(startTime).Nanoseconds()/1000), 1)
        }

        return err
}

func (s *Store) persistFooterUnsynced(file File, footer *Footer) error {
        jBuf, err := json.Marshal(footer)
        if err != nil {
                return err
        }

        finfo, err := file.Stat()
        if err != nil {
                return err
        }

        footerPos := pageAlign(finfo.Size())
        footerLen := footerBegLen + len(jBuf) + footerEndLen

        footerBuf := bytes.NewBuffer(make([]byte, 0, footerLen))
        footerBuf.Write(STORE_MAGIC_BEG)
        footerBuf.Write(STORE_MAGIC_BEG)
        binary.Write(footerBuf, STORE_ENDIAN, uint32(STORE_VERSION))
        binary.Write(footerBuf, STORE_ENDIAN, uint32(footerLen))
        footerBuf.Write(jBuf)
        binary.Write(footerBuf, STORE_ENDIAN, footerPos)
        binary.Write(footerBuf, STORE_ENDIAN, uint32(footerLen))
        footerBuf.Write(STORE_MAGIC_END)
        footerBuf.Write(STORE_MAGIC_END)

        footerWritten, err := file.WriteAt(footerBuf.Bytes(), footerPos)
        if err != nil {
                return err
        }
        if footerWritten != len(footerBuf.Bytes()) {
                return fmt.Errorf("store: persistFooter error writing all footerBuf")
        }

        footer.fileName = finfo.Name()
        footer.filePos = footerPos

        return nil
}

// --------------------------------------------------------

// ReadFooter reads the last valid Footer from a file.
func ReadFooter(options *StoreOptions, file File) (*Footer, error) {
        finfo, err := file.Stat()
        if err != nil {
                return nil, err
        }

        fref := &FileRef{file: file, refs: 1}

        f, err := ScanFooter(options, fref, finfo.Name(), finfo.Size())
        if err != nil {
                return nil, err
        }

        fref.DecRef() // ScanFooter added its own ref-counts on success.

        return f, err
}

// --------------------------------------------------------

// ScanFooter scans a file backwards from the given pos for a valid
// Footer, adding ref-counts to fref on success.
func ScanFooter(options *StoreOptions, fref *FileRef, fileName string,
        pos int64) (*Footer, error) {
        footerEnd := make([]byte, footerEndLen)
        for {
                for { // Scan backwards for STORE_MAGIC_END, which might be a potential footer.
                        if pos <= int64(footerBegLen+footerEndLen) {
                                return nil, ErrNoValidFooter
                        }
                        n, err := fref.file.ReadAt(footerEnd, pos-int64(footerEndLen))
                        if err != nil {
                                return nil, err
                        }
                        if n != footerEndLen {
                                return nil, fmt.Errorf("store: read footer too small")
                        }
                        if bytes.Equal(STORE_MAGIC_END, footerEnd[8+4:8+4+lenMagicEnd]) &&
                                bytes.Equal(STORE_MAGIC_END, footerEnd[8+4+lenMagicEnd:]) {
                                break
                        }

                        pos-- // TODO: optimizations to scan backwards faster.
                }

                // Read and check the potential footer.
                footerEndBuf := bytes.NewBuffer(footerEnd)

                var offset int64
                if err := binary.Read(footerEndBuf, STORE_ENDIAN, &offset); err != nil {
                        return nil, err
                }

                var length uint32
                if err := binary.Read(footerEndBuf, STORE_ENDIAN, &length); err != nil {
                        return nil, err
                }

                if offset > 0 &&
                        offset < pos-int64(footerBegLen+footerEndLen) &&
                        length == uint32(pos-offset) {
                        data := make([]byte, pos-offset-int64(footerEndLen))

                        n, err := fref.file.ReadAt(data, offset)
                        if err != nil {
                                return nil, err
                        }
                        if n != len(data) {
                                return nil, fmt.Errorf("store: read footer data too small")
                        }

                        if bytes.Equal(STORE_MAGIC_BEG, data[:lenMagicBeg]) &&
                                bytes.Equal(STORE_MAGIC_BEG, data[lenMagicBeg:2*lenMagicBeg]) {
                                b := bytes.NewBuffer(data[2*lenMagicBeg:])

                                var version uint32
                                if err := binary.Read(b, STORE_ENDIAN, &version); err != nil {
                                        return nil, err
                                }
                                if version != STORE_VERSION {
                                        return nil, fmt.Errorf("store: version mismatch, "+
                                                "current: %v != found: %v", STORE_VERSION, version)
                                }

                                var length0 uint32
                                if err := binary.Read(b, STORE_ENDIAN, &length0); err != nil {
                                        return nil, err
                                }
                                if length0 != length {
                                        return nil, fmt.Errorf("store: length mismatch, "+
                                                "wanted: %v != found: %v", length0, length)
                                }

                                f := &Footer{refs: 1, fileName: fileName, filePos: offset}

                                err = json.Unmarshal(data[2*lenMagicBeg+4+4:], f)
                                if err != nil {
                                        return nil, err
                                }

                                // json.Unmarshal would have just loaded the map.
                                // We now need to load the segments into the map.
                                // Recursively load the Child Footer segment stacks.
                                err = f.loadSegments(options, fref)
                                if err != nil {
                                        return nil, err
                                }

                                return f, nil
                        } // Else, perhaps file was unlucky in having STORE_MAGIC_END's.
                } // Else, perhaps a persist file was stored in a file.

                pos-- // Footer was invalid, so keep scanning.
        }
}

// --------------------------------------------------------

// loadSegments() loads the segments of a footer, which must not be
// already loaded.  The footer adds new ref-counts to the fref on
// success.  The footer will be in an already closed state on error.
func (f *Footer) loadSegments(options *StoreOptions, fref *FileRef) (err error) {
        // Track mrefs that we need to DecRef() if there's an error.
        mrefs := make([]*mmapRef, 0, len(f.SegmentLocs))
        mrefs, err = f.doLoadSegments(options, fref, mrefs)
        if err != nil {
                for _, mref := range mrefs {
                        mref.DecRef()
                }
                return err
        }

        return err
}

func (f *Footer) doLoadSegments(options *StoreOptions, fref *FileRef,
        mrefs []*mmapRef) (mrefsSoFar []*mmapRef, err error) {
        // Recursively load the childFooters first..
        for _, childFooter := range f.ChildFooters {
                mrefs, err = childFooter.doLoadSegments(options, fref, mrefs)
                if err != nil {
                        return mrefs, err
                }
        }

        if f.ss != nil && f.ss.a != nil {
                return mrefs, nil
        }

        osFile := ToOsFile(fref.file)
        if osFile == nil {
                return mrefs, fmt.Errorf("store: loadSegments convert to os.File error")
        }

        a := make([]Segment, len(f.SegmentLocs))

        for i := range f.SegmentLocs {
                sloc := &f.SegmentLocs[i]

                mref := sloc.mref
                if mref != nil {
                        if mref.fref != fref {
                                return mrefs, fmt.Errorf("store: loadSegments fref mismatch")
                        }

                        mref.AddRef()
                } else {
                        // We persist kvs before buf, so KvsOffset < BufOffset.
                        begOffset := int64(sloc.KvsOffset)
                        endOffset := int64(sloc.BufOffset + sloc.BufBytes)

                        nbytes := int(endOffset - begOffset)

                        // Some platforms (windows) only support mmap()'ing at an
                        // allocation granularity that's != to a page size, so
                        // calculate the actual offset/nbytes to use.
                        begOffsetActual := pageOffset(begOffset, int64(AllocationGranularity))
                        begOffsetDelta := int(begOffset - begOffsetActual)
                        nbytesActual := nbytes + begOffsetDelta

                        mm, err := mmap.MapRegion(osFile, nbytesActual, mmap.RDONLY, 0, begOffsetActual)
                        if err != nil {
                                return mrefs, fmt.Errorf("store: loadSegments mmap.Map(), err: %v", err)
                        }

                        fref.AddRef() // New mref owns 1 fref ref-count.

                        buf := mm[begOffsetDelta : begOffsetDelta+nbytes]

                        sloc.mref = &mmapRef{fref: fref, mm: mm, buf: buf, refs: 1}

                        mref = sloc.mref
                }

                mrefs = append(mrefs, mref)

                segmentLoader, exists := SegmentLoaders[sloc.Kind]
                if !exists || segmentLoader == nil {
                        return mrefs, fmt.Errorf("store: unknown SegmentLoc kind, sloc: %+v", sloc)
                }

                var kvs []uint64
                var buf []byte

                if sloc.KvsBytes > 0 {
                        if sloc.KvsBytes > uint64(len(mref.buf)) {
                                return mrefs, fmt.Errorf("store_footer: KvsOffset/KvsBytes too big,"+
                                        " len(mref.buf): %d, sloc: %+v, footer: %+v,"+
                                        " f.SegmentLocs: %+v, i: %d, options: %v",
                                        len(mref.buf), sloc, f, f.SegmentLocs, i, options)
                        }

                        kvsBytes := mref.buf[0:sloc.KvsBytes]
                        kvs, err = ByteSliceToUint64Slice(kvsBytes)
                        if err != nil {
                                return mrefs, err
                        }
                }

                if sloc.BufBytes > 0 {
                        bufStart := sloc.BufOffset - sloc.KvsOffset
                        if bufStart+sloc.BufBytes > uint64(len(mref.buf)) {
                                return mrefs, fmt.Errorf("store_footer: BufOffset/BufBytes too big,"+
                                        " len(mref.buf): %d, sloc: %+v, footer: %+v,"+
                                        " f.SegmentLocs: %+v, i: %d, options: %v",
                                        len(mref.buf), sloc, f, f.SegmentLocs, i, options)
                        }

                        buf = mref.buf[bufStart : bufStart+sloc.BufBytes]
                }

                seg, err := segmentLoader(sloc, kvs, buf)
                if err != nil {
                        return mrefs, err
                }

                a[i] = seg
        }

        f.ss = &segmentStack{
                options: &options.CollectionOptions,
                a:       a,
                refs:    1,
        }

        return mrefs, nil
}

// --------------------------------------------------------

// ChildCollectionNames returns an array of child collection name strings.
func (f *Footer) ChildCollectionNames() ([]string, error) {
        var childNames = make([]string, len(f.ChildFooters))
        idx := 0
        for name, _ := range f.ChildFooters {
                childNames[idx] = name
                idx++
        }
        return childNames, nil
}

// ChildCollectionSnapshot returns a Snapshot on a given child
// collection by its name.
func (f *Footer) ChildCollectionSnapshot(childCollectionName string) (
        Snapshot, error) {
        childFooter, exists := f.ChildFooters[childCollectionName]
        if !exists {
                return nil, nil
        }
        childFooter.AddRef()
        return childFooter, nil
}

func (f *Footer) Close() error {
        f.DecRef()
        return nil
}

func (f *Footer) AddRef() {
        f.m.Lock()
        f.refs++
        f.m.Unlock()
}

func (f *Footer) DecRef() {
        f.m.Lock()
        f.refs--
        if f.refs <= 0 {
                f.SegmentLocs.DecRef()
                f.SegmentLocs = nil
                f.ss = nil
        }
        f.m.Unlock()
}

func (f *Footer) Length() uint64 {
        jBuf, err := json.Marshal(f)
        if err != nil {
                return 0
        }

        footerLen := footerBegLen + len(jBuf) + footerEndLen
        return uint64(footerLen)
}

// --------------------------------------------------------

// SegmentStack() returns the current SegmentLocs and segmentStack for
// a footer, while also incrementing the ref-count on the footer.  The
// caller must DecRef() the footer when done.
func (f *Footer) SegmentStack() (SegmentLocs, *segmentStack) {
        f.m.Lock()

        f.refs++

        slocs, ss := f.SegmentLocs, f.ss

        f.m.Unlock()

        return slocs, ss
}

// --------------------------------------------------------

// Get retrieves a val from the footer, and will return nil val
// if the entry does not exist in the footer.
func (f *Footer) Get(key []byte, readOptions ReadOptions) ([]byte, error) {
        _, ss := f.SegmentStack()
        if ss == nil {
                f.DecRef()
                return nil, nil
        }

        rv, err := ss.Get(key, readOptions)
        if err == nil && rv != nil && !readOptions.NoCopyValue {
                rv = append(make([]byte, 0, len(rv)), rv...) // Copy.
        }

        f.DecRef()

        return rv, err
}

// StartIterator returns a new Iterator instance on this footer.
//
// On success, the returned Iterator will be positioned so that
// Iterator.Current() will either provide the first entry in the
// range or ErrIteratorDone.
//
// A startKeyIncl of nil means the logical "bottom-most" possible key
// and an endKeyExcl of nil means the logical "top-most" possible key.
func (f *Footer) StartIterator(startKeyIncl, endKeyExcl []byte,
        iteratorOptions IteratorOptions) (Iterator, error) {
        _, ss := f.SegmentStack()
        if ss == nil {
                f.DecRef()
                return nil, nil
        }

        iter, err := ss.StartIterator(startKeyIncl, endKeyExcl, iteratorOptions)
        if err != nil || iter == nil {
                f.DecRef()
                return nil, err
        }

        initCloser, ok := iter.(InitCloser)
        if !ok || initCloser == nil {
                iter.Close()
                f.DecRef()
                return nil, ErrUnexpected
        }

        err = initCloser.InitCloser(f)
        if err != nil {
                iter.Close()
                f.DecRef()
                return nil, err
        }

        return iter, nil
}

1	// Copyright (c) 2016 Couchbase, Inc.
2	// Licensed under the Apache License, Version 2.0 (the "License");
3	// you may not use this file except in compliance with the
4	// License. You may obtain a copy of the License at
5	// http://www.apache.org/licenses/LICENSE-2.0
6	// Unless required by applicable law or agreed to in writing,
7	// software distributed under the License is distributed on an "AS
8	// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
9	// express or implied. See the License for the specific language
10	// governing permissions and limitations under the License.
11
12	package moss
13
14	import (
15	"bytes"
16	"encoding/binary"
17	"encoding/json"
18	"fmt"
19	"time"
20
21	"github.com/edsrzf/mmap-go"
22	)
23
24	func (s Store) persistFooter(file File, footer Footer,
25	options StorePersistOptions) error {	137✔
26	startTime := time.Now()	137✔
27	if !options.NoSync {	273✔
28	err := file.Sync()	136✔
29	if err != nil {	136✔
30	return err	×
31	}	×
32	}
33
34	err := s.persistFooterUnsynced(file, footer)	137✔
35	if err != nil {	137✔
36	return err	×
37	}	×
38
39	if !options.NoSync {	273✔
40	err = file.Sync()	136✔
41	}	136✔
42
43	if err == nil {	274✔
44	s.histograms["PersistFooterUsecs"].Add(	137✔
45	uint64(time.Since(startTime).Nanoseconds()/1000), 1)	137✔
46	}	137✔
47
48	return err	137✔
49	}
50
51	func (s Store) persistFooterUnsynced(file File, footer Footer) error {	137✔
52	jBuf, err := json.Marshal(footer)	137✔
53	if err != nil {	137✔
54	return err	×
55	}	×
56
57	finfo, err := file.Stat()	137✔
58	if err != nil {	137✔
59	return err	×
60	}	×
61
62	footerPos := pageAlign(finfo.Size())	137✔
63	footerLen := footerBegLen + len(jBuf) + footerEndLen	137✔
64		137✔
65	footerBuf := bytes.NewBuffer(make([]byte, 0, footerLen))	137✔
66	footerBuf.Write(STORE_MAGIC_BEG)	137✔
67	footerBuf.Write(STORE_MAGIC_BEG)	137✔
68	binary.Write(footerBuf, STORE_ENDIAN, uint32(STORE_VERSION))	137✔
69	binary.Write(footerBuf, STORE_ENDIAN, uint32(footerLen))	137✔
70	footerBuf.Write(jBuf)	137✔
71	binary.Write(footerBuf, STORE_ENDIAN, footerPos)	137✔
72	binary.Write(footerBuf, STORE_ENDIAN, uint32(footerLen))	137✔
73	footerBuf.Write(STORE_MAGIC_END)	137✔
74	footerBuf.Write(STORE_MAGIC_END)	137✔
75		137✔
76	footerWritten, err := file.WriteAt(footerBuf.Bytes(), footerPos)	137✔
77	if err != nil {	137✔
78	return err	×
79	}	×
80	if footerWritten != len(footerBuf.Bytes()) {	137✔
81	return fmt.Errorf("store: persistFooter error writing all footerBuf")	×
82	}	×
83
84	footer.fileName = finfo.Name()	137✔
85	footer.filePos = footerPos	137✔
86		137✔
87	return nil	137✔
88	}
89
90	// --------------------------------------------------------
91
92	// ReadFooter reads the last valid Footer from a file.
93	func ReadFooter(options StoreOptions, file File) (Footer, error) {	128✔
94	finfo, err := file.Stat()	128✔
95	if err != nil {	128✔
96	return nil, err	×
97	}	×
98
99	fref := &FileRef{file: file, refs: 1}	128✔
100		128✔
101	f, err := ScanFooter(options, fref, finfo.Name(), finfo.Size())	128✔
102	if err != nil {	132✔
103	return nil, err	4✔
104	}	4✔
105
106	fref.DecRef() // ScanFooter added its own ref-counts on success.	124✔
107		124✔
108	return f, err	124✔
109	}
110
111	// --------------------------------------------------------
112
113	// ScanFooter scans a file backwards from the given pos for a valid
114	// Footer, adding ref-counts to fref on success.
115	func ScanFooter(options StoreOptions, fref FileRef, fileName string,
116	pos int64) (*Footer, error) {	156✔
117	footerEnd := make([]byte, footerEndLen)	156✔
118	for {	312✔
119	for { // Scan backwards for STORE_MAGIC_END, which might be a potential footer.	356,589✔
120	if pos <= int64(footerBegLen+footerEndLen) {	356,450✔
121	return nil, ErrNoValidFooter	17✔
122	}	17✔
123	n, err := fref.file.ReadAt(footerEnd, pos-int64(footerEndLen))	356,416✔
124	if err != nil {	356,416✔
125	return nil, err	×
126	}	×
127	if n != footerEndLen {	356,416✔
128	return nil, fmt.Errorf("store: read footer too small")	×
129	}	×
130	if bytes.Equal(STORE_MAGIC_END, footerEnd[8+4:8+4+lenMagicEnd]) &&	356,416✔
131	bytes.Equal(STORE_MAGIC_END, footerEnd[8+4+lenMagicEnd:]) {	356,555✔
132	break	139✔
133	}
134
135	pos-- // TODO: optimizations to scan backwards faster.	356,277✔
136	}
137
138	// Read and check the potential footer.
139	footerEndBuf := bytes.NewBuffer(footerEnd)	139✔
140		139✔
141	var offset int64	139✔
142	if err := binary.Read(footerEndBuf, STORE_ENDIAN, &offset); err != nil {	139✔
143	return nil, err	×
144	}	×
145
146	var length uint32	139✔
147	if err := binary.Read(footerEndBuf, STORE_ENDIAN, &length); err != nil {	139✔
148	return nil, err	×
149	}	×
150
151	if offset > 0 &&	139✔
152	offset < pos-int64(footerBegLen+footerEndLen) &&	139✔
153	length == uint32(pos-offset) {	278✔
154	data := make([]byte, pos-offset-int64(footerEndLen))	139✔
155		139✔
156	n, err := fref.file.ReadAt(data, offset)	139✔
157	if err != nil {	139✔
158	return nil, err	×
159	}	×
160	if n != len(data) {	139✔
161	return nil, fmt.Errorf("store: read footer data too small")	×
162	}	×
163
164	if bytes.Equal(STORE_MAGIC_BEG, data[:lenMagicBeg]) &&	139✔
165	bytes.Equal(STORE_MAGIC_BEG, data[lenMagicBeg:2*lenMagicBeg]) {	278✔
166	b := bytes.NewBuffer(data[2*lenMagicBeg:])	139✔
167		139✔
168	var version uint32	139✔
169	if err := binary.Read(b, STORE_ENDIAN, &version); err != nil {	139✔
170	return nil, err	×
171	}	×
172	if version != STORE_VERSION {	139✔
173	return nil, fmt.Errorf("store: version mismatch, "+	×
174	"current: %v != found: %v", STORE_VERSION, version)	×
175	}	×
176
177	var length0 uint32	139✔
178	if err := binary.Read(b, STORE_ENDIAN, &length0); err != nil {	139✔
179	return nil, err	×
180	}	×
181	if length0 != length {	139✔
182	return nil, fmt.Errorf("store: length mismatch, "+	×
183	"wanted: %v != found: %v", length0, length)	×
184	}	×
185
186	f := &Footer{refs: 1, fileName: fileName, filePos: offset}	139✔
187		139✔
188	err = json.Unmarshal(data[2*lenMagicBeg+4+4:], f)	139✔
189	if err != nil {	139✔
190	return nil, err	×
191	}	×
192
193	// json.Unmarshal would have just loaded the map.
194	// We now need to load the segments into the map.
195	// Recursively load the Child Footer segment stacks.
196	err = f.loadSegments(options, fref)	139✔
197	if err != nil {	139✔
198	return nil, err	×
199	}	×
200
201	return f, nil	139✔
202	} // Else, perhaps file was unlucky in having STORE_MAGIC_END's.
203	} // Else, perhaps a persist file was stored in a file.
204
205	pos-- // Footer was invalid, so keep scanning.	×
206	}
207	}
208
209	// --------------------------------------------------------
210
211	// loadSegments() loads the segments of a footer, which must not be
212	// already loaded. The footer adds new ref-counts to the fref on
213	// success. The footer will be in an already closed state on error.
214	func (f Footer) loadSegments(options StoreOptions, fref *FileRef) (err error) {	202✔
215	// Track mrefs that we need to DecRef() if there's an error.	202✔
216	mrefs := make([]*mmapRef, 0, len(f.SegmentLocs))	202✔
217	mrefs, err = f.doLoadSegments(options, fref, mrefs)	202✔
218	if err != nil {	202✔
219	for _, mref := range mrefs {	×
220	mref.DecRef()	×
221	}	×
222	return err	×
223	}
224
225	return err	202✔
226	}
227
228	func (f Footer) doLoadSegments(options StoreOptions, fref *FileRef,
229	mrefs []mmapRef) (mrefsSoFar []mmapRef, err error) {	247✔
230	// Recursively load the childFooters first..	247✔
231	for _, childFooter := range f.ChildFooters {	292✔
232	mrefs, err = childFooter.doLoadSegments(options, fref, mrefs)	45✔
233	if err != nil {	45✔
234	return mrefs, err	×
235	}	×
236	}
237
238	if f.ss != nil && f.ss.a != nil {	247✔
239	return mrefs, nil	×
240	}	×
241
242	osFile := ToOsFile(fref.file)	247✔
243	if osFile == nil {	247✔
244	return mrefs, fmt.Errorf("store: loadSegments convert to os.File error")	×
245	}	×
246
247	a := make([]Segment, len(f.SegmentLocs))	247✔
248		247✔
249	for i := range f.SegmentLocs {	696✔
250	sloc := &f.SegmentLocs[i]	449✔
251		449✔
252	mref := sloc.mref	449✔
253	if mref != nil {	622✔
254	if mref.fref != fref {	173✔
255	return mrefs, fmt.Errorf("store: loadSegments fref mismatch")	×
256	}	×
257
258	mref.AddRef()	173✔
259	} else {	276✔
260	// We persist kvs before buf, so KvsOffset < BufOffset.	276✔
261	begOffset := int64(sloc.KvsOffset)	276✔
262	endOffset := int64(sloc.BufOffset + sloc.BufBytes)	276✔
263		276✔
264	nbytes := int(endOffset - begOffset)	276✔
265		276✔
266	// Some platforms (windows) only support mmap()'ing at an	276✔
267	// allocation granularity that's != to a page size, so	276✔
268	// calculate the actual offset/nbytes to use.	276✔
269	begOffsetActual := pageOffset(begOffset, int64(AllocationGranularity))	276✔
270	begOffsetDelta := int(begOffset - begOffsetActual)	276✔
271	nbytesActual := nbytes + begOffsetDelta	276✔
272		276✔
273	mm, err := mmap.MapRegion(osFile, nbytesActual, mmap.RDONLY, 0, begOffsetActual)	276✔
274	if err != nil {	276✔
275	return mrefs, fmt.Errorf("store: loadSegments mmap.Map(), err: %v", err)	×
276	}	×
277
278	fref.AddRef() // New mref owns 1 fref ref-count.	276✔
279		276✔
280	buf := mm[begOffsetDelta : begOffsetDelta+nbytes]	276✔
281		276✔
282	sloc.mref = &mmapRef{fref: fref, mm: mm, buf: buf, refs: 1}	276✔
283		276✔
284	mref = sloc.mref	276✔
285	}
286
287	mrefs = append(mrefs, mref)	449✔
288		449✔
289	segmentLoader, exists := SegmentLoaders[sloc.Kind]	449✔
290	if !exists \|\| segmentLoader == nil {	449✔
291	return mrefs, fmt.Errorf("store: unknown SegmentLoc kind, sloc: %+v", sloc)	×
292	}	×
293
294	var kvs []uint64	449✔
295	var buf []byte	449✔
296		449✔
297	if sloc.KvsBytes > 0 {	892✔
298	if sloc.KvsBytes > uint64(len(mref.buf)) {	443✔
299	return mrefs, fmt.Errorf("store_footer: KvsOffset/KvsBytes too big,"+	×
300	" len(mref.buf): %d, sloc: %+v, footer: %+v,"+	×
301	" f.SegmentLocs: %+v, i: %d, options: %v",	×
302	len(mref.buf), sloc, f, f.SegmentLocs, i, options)	×
303	}	×
304
305	kvsBytes := mref.buf[0:sloc.KvsBytes]	443✔
306	kvs, err = ByteSliceToUint64Slice(kvsBytes)	443✔
307	if err != nil {	443✔
308	return mrefs, err	×
309	}	×
310	}
311
312	if sloc.BufBytes > 0 {	892✔
313	bufStart := sloc.BufOffset - sloc.KvsOffset	443✔
314	if bufStart+sloc.BufBytes > uint64(len(mref.buf)) {	443✔
315	return mrefs, fmt.Errorf("store_footer: BufOffset/BufBytes too big,"+	×
316	" len(mref.buf): %d, sloc: %+v, footer: %+v,"+	×
317	" f.SegmentLocs: %+v, i: %d, options: %v",	×
318	len(mref.buf), sloc, f, f.SegmentLocs, i, options)	×
319	}	×
320
321	buf = mref.buf[bufStart : bufStart+sloc.BufBytes]	443✔
322	}
323
324	seg, err := segmentLoader(sloc, kvs, buf)	449✔
325	if err != nil {	449✔
326	return mrefs, err	×
327	}	×
328
329	a[i] = seg	449✔
330	}
331
332	f.ss = &segmentStack{	247✔
333	options: &options.CollectionOptions,	247✔
334	a: a,	247✔
335	refs: 1,	247✔
336	}	247✔
337		247✔
338	return mrefs, nil	247✔
339	}
340
341	// --------------------------------------------------------
342
343	// ChildCollectionNames returns an array of child collection name strings.
344	func (f *Footer) ChildCollectionNames() ([]string, error) {	×
345	var childNames = make([]string, len(f.ChildFooters))	×
346	idx := 0	×
347	for name, _ := range f.ChildFooters {	×
348	childNames[idx] = name	×
349	idx++	×
350	}	×
351	return childNames, nil	×
352	}
353
354	// ChildCollectionSnapshot returns a Snapshot on a given child
355	// collection by its name.
356	func (f *Footer) ChildCollectionSnapshot(childCollectionName string) (
357	Snapshot, error) {	368✔
358	childFooter, exists := f.ChildFooters[childCollectionName]	368✔
359	if !exists {	409✔
360	return nil, nil	41✔
361	}	41✔
362	childFooter.AddRef()	327✔
363	return childFooter, nil	327✔
364	}
365
366	func (f *Footer) Close() error {	560✔
367	f.DecRef()	560✔
368	return nil	560✔
369	}	560✔
370
371	func (f *Footer) AddRef() {	635✔
372	f.m.Lock()	635✔
373	f.refs++	635✔
374	f.m.Unlock()	635✔
375	}	635✔
376
377	func (f *Footer) DecRef() {	3,714✔
378	f.m.Lock()	3,714✔
379	f.refs--	3,714✔
380	if f.refs <= 0 {	3,964✔
381	f.SegmentLocs.DecRef()	250✔
382	f.SegmentLocs = nil	250✔
383	f.ss = nil	250✔
384	}	250✔
385	f.m.Unlock()	3,714✔
386	}
387
388	func (f *Footer) Length() uint64 {	3✔
389	jBuf, err := json.Marshal(f)	3✔
390	if err != nil {	3✔
391	return 0	×
392	}	×
393
394	footerLen := footerBegLen + len(jBuf) + footerEndLen	3✔
395	return uint64(footerLen)	3✔
396	}
397
398	// --------------------------------------------------------
399
400	// SegmentStack() returns the current SegmentLocs and segmentStack for
401	// a footer, while also incrementing the ref-count on the footer. The
402	// caller must DecRef() the footer when done.
403	func (f Footer) SegmentStack() (SegmentLocs, segmentStack) {	3,160✔
404	f.m.Lock()	3,160✔
405		3,160✔
406	f.refs++	3,160✔
407		3,160✔
408	slocs, ss := f.SegmentLocs, f.ss	3,160✔
409		3,160✔
410	f.m.Unlock()	3,160✔
411		3,160✔
412	return slocs, ss	3,160✔
413	}	3,160✔
414
415	// --------------------------------------------------------
416
417	// Get retrieves a val from the footer, and will return nil val
418	// if the entry does not exist in the footer.
419	func (f *Footer) Get(key []byte, readOptions ReadOptions) ([]byte, error) {	2,685✔
420	_, ss := f.SegmentStack()	2,685✔
421	if ss == nil {	2,685✔
422	f.DecRef()	×
423	return nil, nil	×
424	}	×
425
426	rv, err := ss.Get(key, readOptions)	2,685✔
427	if err == nil && rv != nil && !readOptions.NoCopyValue {	4,697✔
428	rv = append(make([]byte, 0, len(rv)), rv...) // Copy.	2,012✔
429	}	2,012✔
430
431	f.DecRef()	2,685✔
432		2,685✔
433	return rv, err	2,685✔
434	}
435
436	// StartIterator returns a new Iterator instance on this footer.
437	//
438	// On success, the returned Iterator will be positioned so that
439	// Iterator.Current() will either provide the first entry in the
440	// range or ErrIteratorDone.
441	//
442	// A startKeyIncl of nil means the logical "bottom-most" possible key
443	// and an endKeyExcl of nil means the logical "top-most" possible key.
444	func (f *Footer) StartIterator(startKeyIncl, endKeyExcl []byte,
445	iteratorOptions IteratorOptions) (Iterator, error) {	227✔
446	_, ss := f.SegmentStack()	227✔
447	if ss == nil {	227✔
448	f.DecRef()	×
449	return nil, nil	×
450	}	×
451
452	iter, err := ss.StartIterator(startKeyIncl, endKeyExcl, iteratorOptions)	227✔
453	if err != nil \|\| iter == nil {	227✔
454	f.DecRef()	×
455	return nil, err	×
456	}	×
457
458	initCloser, ok := iter.(InitCloser)	227✔
459	if !ok \|\| initCloser == nil {	227✔
460	iter.Close()	×
461	f.DecRef()	×
462	return nil, ErrUnexpected	×
463	}	×
464
465	err = initCloser.InitCloser(f)	227✔
466	if err != nil {	227✔
467	iter.Close()	×
468	f.DecRef()	×
469	return nil, err	×
470	}	×
471
472	return iter, nil	227✔
473	}

couchbase / moss / 194

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous