couchbase / moss / 254

pending completion

Build # 254

Build Type

push

travis-ci

Committed by

hisundar

Commit Message

<a href="https://github.com/couchbase/moss/commit/<a class=hub.com/couchbase/moss/commit/<a class="double-link" href="https://git"><a class=hub.com/couchbase/moss/commit/<a class="double-link" href="https://git"><a class=hub.com/couchbase/moss/commit/<a class="double-link" href="https://git"><a class=hub.com/couchbase/moss/commit/c44392bd19c1b1ff67508311d69f99c44cec79ad">c44392bd1<a href="https://github.com/couchbase/moss/commit/c44392bd19c1b1ff67508311d69f99c44cec79ad">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/couchbase/moss/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/couchbase/moss/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/couchbase/moss/commit/c44392bd19c1b1ff67508311d69f99c44cec79ad&quot;&gt;c44392bd1&lt;/a&gt;&lt;a href=&quot;https://github.com/couchbase/moss/commit/c44392bd19c1b1ff67508311d69f99c44cec79ad&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/couchbase/moss/commit/c44392bd19c1b1ff67508311d69f99c44cec79ad&amp;quot;&amp;gt;&amp;amp;quot;&amp;amp;gt;MB-24903: Don&amp;amp;amp;#39;t ScanFooter from End-Of-File

If file size happens to be 4K aligned, ScanFooter
tries to read from EOF, fails and refuses to open store.
To avoid this bug, start scanning footer from at least
1 byte before end of file.

Change-Id: I&amp;lt;/a&amp;gt;&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/couchbase/moss/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/couchbase/moss/commit/<a class="double-link" href="https://github.com/couchbase/moss/commit/e45c8c86280a9772b00fdb4202ba88bc9049d6eb">e45c8c862</a>&quot;&gt;e45c8c862&lt;/a&gt;&quot;&gt;e45c8c862&lt;/a&gt;&lt;a href=&quot;https://github.com/couchbase/moss/commit/c44392bd19c1b1ff67508311d69f99c44cec79ad&quot;</a>>
Reviewed-on: http://review.couchbase.org/79623
Well-Formed: Build Bot &lt;build@couchbase.com&gt;
Reviewed-by: Steve Yen &lt;steve.yen@gmail.com>
Tested-by: Sundararaman Sridharan <sundar@couchbase.com>

Run Details

4 of 4 new or added lines in 1 file covered. (100.0%)

2977 of 3586 relevant lines covered (83.02%)

84267.61 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

70.38

/store_footer.go

//  Copyright (c) 2016 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package moss

import (
        "bytes"
        "encoding/binary"
        "encoding/json"
        "fmt"
        "time"

        "github.com/edsrzf/mmap-go"
)

func (s *Store) persistFooter(file File, footer *Footer,
        options StorePersistOptions) error {
        startTime := time.Now()

        if !options.NoSync {
                err := file.Sync()
                if err != nil {
                        return err
                }
        }

        err := s.persistFooterUnsynced(file, footer)
        if err != nil {
                return err
        }

        if !options.NoSync {
                err = file.Sync()
        }

        if err == nil {
                s.histograms["PersistFooterUsecs"].Add(
                        uint64(time.Since(startTime).Nanoseconds()/1000), 1)
        }

        return err
}

func (s *Store) persistFooterUnsynced(file File, footer *Footer) error {
        jBuf, err := json.Marshal(footer)
        if err != nil {
                return err
        }

        finfo, err := file.Stat()
        if err != nil {
                return err
        }

        footerPos := pageAlignCeil(finfo.Size())
        footerLen := footerBegLen + len(jBuf) + footerEndLen

        footerBuf := bytes.NewBuffer(make([]byte, 0, footerLen))
        footerBuf.Write(StoreMagicBeg)
        footerBuf.Write(StoreMagicBeg)
        binary.Write(footerBuf, StoreEndian, uint32(StoreVersion))
        binary.Write(footerBuf, StoreEndian, uint32(footerLen))
        footerBuf.Write(jBuf)
        binary.Write(footerBuf, StoreEndian, footerPos)
        binary.Write(footerBuf, StoreEndian, uint32(footerLen))
        footerBuf.Write(StoreMagicEnd)
        footerBuf.Write(StoreMagicEnd)

        footerWritten, err := file.WriteAt(footerBuf.Bytes(), footerPos)
        if err != nil {
                return err
        }
        if footerWritten != len(footerBuf.Bytes()) {
                return fmt.Errorf("store: persistFooter error writing all footerBuf")
        }

        footer.fileName = finfo.Name()
        footer.filePos = footerPos

        return nil
}

// --------------------------------------------------------

// ReadFooter reads the last valid Footer from a file.
func ReadFooter(options *StoreOptions, file File) (*Footer, error) {
        finfo, err := file.Stat()
        if err != nil {
                return nil, err
        }

        fref := &FileRef{file: file, refs: 1}

        // To avoid an EOF while reading, start scanning the footer from
        // the last byte. This is under the assumption that the footer is
        // at least 2 bytes long.
        f, err := ScanFooter(options, fref, finfo.Name(), finfo.Size()-1)
        if err != nil {
                return nil, err
        }

        fref.DecRef() // ScanFooter added its own ref-counts on success.

        return f, err
}

// --------------------------------------------------------

// ScanFooter scans a file backwards from the given pos for a valid
// Footer, adding ref-counts to fref on success.
func ScanFooter(options *StoreOptions, fref *FileRef, fileName string,
        pos int64) (*Footer, error) {
        footerBeg := make([]byte, footerBegLen)

        // Align pos to the start of a page (floor).
        pos = pageAlignFloor(pos)

        for {
                for { // Scan for StoreMagicBeg, which may be a potential footer.
                        if pos <= 0 {
                                return nil, ErrNoValidFooter
                        }

                        n, err := fref.file.ReadAt(footerBeg, pos)
                        if err != nil {
                                return nil, err
                        }

                        if n == footerBegLen &&
                                bytes.Equal(StoreMagicBeg, footerBeg[:lenMagicBeg]) &&
                                bytes.Equal(StoreMagicBeg, footerBeg[lenMagicBeg:2*lenMagicBeg]) {
                                break
                        }

                        // Move pos back by page size.
                        pos -= int64(StorePageSize)
                }

                // Read and check the potential footer.
                footerBegBuf := bytes.NewBuffer(footerBeg[2*lenMagicBeg:])

                var version uint32
                if err := binary.Read(footerBegBuf, StoreEndian, &version); err != nil {
                        return nil, err
                }
                if version != StoreVersion {
                        return nil, fmt.Errorf("store: version mismatch, "+
                                "current: %v != found: %v", StoreVersion, version)
                }

                var length uint32
                if err := binary.Read(footerBegBuf, StoreEndian, &length); err != nil {
                        return nil, err
                }

                data := make([]byte, int64(length)-int64(footerBegLen))

                n, err := fref.file.ReadAt(data, pos+int64(footerBegLen))
                if err != nil {
                        return nil, err
                }

                if n == len(data) &&
                        bytes.Equal(StoreMagicEnd, data[n-lenMagicEnd*2:n-lenMagicEnd]) &&
                        bytes.Equal(StoreMagicEnd, data[n-lenMagicEnd:]) {

                        content := int(length) - footerBegLen - footerEndLen
                        b := bytes.NewBuffer(data[content:])

                        var offset int64
                        if err = binary.Read(b, StoreEndian, &offset); err != nil {
                                return nil, err
                        }
                        if offset != pos {
                                return nil, fmt.Errorf("store: offset mismatch, "+
                                        "wanted: %v != found: %v", offset, pos)
                        }

                        var length1 uint32
                        if err = binary.Read(b, StoreEndian, &length1); err != nil {
                                return nil, err
                        }
                        if length1 != length {
                                return nil, fmt.Errorf("store: length mismatch, "+
                                        "wanted: %v != found: %v", length1, length)
                        }

                        f := &Footer{refs: 1, fileName: fileName, filePos: offset}

                        err = json.Unmarshal(data[:content], f)
                        if err != nil {
                                return nil, err
                        }

                        // json.Unmarshal would have just loaded the map.
                        // We now need to load each segment into the map.
                        // Also recursively load child footer segment stacks.
                        err = f.loadSegments(options, fref)
                        if err != nil {
                                return nil, err
                        }

                        return f, nil
                }
                // Else, invalid footer - StoreMagicEnd missing and/or file
                // pos out of bounds.

                // Footer was invalid, so keep scanning.
                pos -= int64(StorePageSize)
        }
}

// --------------------------------------------------------

// loadSegments() loads the segments of a footer.  Adds new ref-counts
// to the fref on success.  The footer will be in an already closed
// state on error.
func (f *Footer) loadSegments(options *StoreOptions, fref *FileRef) (err error) {
        // Track mrefs that we need to DecRef() if there's an error.
        mrefs := make([]*mmapRef, 0, len(f.SegmentLocs))
        mrefs, err = f.doLoadSegments(options, fref, mrefs)
        if err != nil {
                for _, mref := range mrefs {
                        mref.DecRef()
                }
                return err
        }

        return nil
}

func (f *Footer) doLoadSegments(options *StoreOptions, fref *FileRef,
        mrefs []*mmapRef) (mrefsSoFar []*mmapRef, err error) {
        // Recursively load the childFooters first.
        for _, childFooter := range f.ChildFooters {
                mrefs, err = childFooter.doLoadSegments(options, fref, mrefs)
                if err != nil {
                        return mrefs, err
                }
        }

        if f.ss != nil && f.ss.a != nil {
                return mrefs, nil
        }

        osFile := ToOsFile(fref.file)
        if osFile == nil {
                return mrefs, fmt.Errorf("store: doLoadSegments convert to os.File error")
        }

        a := make([]Segment, len(f.SegmentLocs))

        for i := range f.SegmentLocs {
                sloc := &f.SegmentLocs[i]

                mref := sloc.mref
                if mref != nil {
                        if mref.fref != fref {
                                return mrefs, fmt.Errorf("store: doLoadSegments fref mismatch")
                        }

                        mref.AddRef()
                } else {
                        // We persist kvs before buf, so KvsOffset < BufOffset.
                        begOffset := int64(sloc.KvsOffset)
                        endOffset := int64(sloc.BufOffset + sloc.BufBytes)

                        nbytes := int(endOffset - begOffset)

                        // Some platforms (windows) only support mmap()'ing at an
                        // allocation granularity that's != to a page size, so
                        // calculate the actual offset/nbytes to use.
                        begOffsetActual := pageOffset(begOffset, int64(AllocationGranularity))
                        begOffsetDelta := int(begOffset - begOffsetActual)
                        nbytesActual := nbytes + begOffsetDelta

                        mm, err := mmap.MapRegion(osFile, nbytesActual, mmap.RDONLY, 0, begOffsetActual)
                        if err != nil {
                                return mrefs, fmt.Errorf("store: doLoadSegments mmap.Map(), err: %v", err)
                        }

                        fref.AddRef() // New mref owns 1 fref ref-count.

                        buf := mm[begOffsetDelta : begOffsetDelta+nbytes]

                        sloc.mref = &mmapRef{fref: fref, mm: mm, buf: buf, refs: 1}

                        mref = sloc.mref
                }

                mrefs = append(mrefs, mref)

                segmentLoader, exists := SegmentLoaders[sloc.Kind]
                if !exists || segmentLoader == nil {
                        return mrefs, fmt.Errorf("store: unknown SegmentLoc kind, sloc: %+v", sloc)
                }

                seg, err := segmentLoader(sloc)
                if err != nil {
                        return mrefs, fmt.Errorf("store: segmentLoader failed, footer: %+v,"+
                                " f.SegmentLocs: %+v, i: %d, options: %v err: %+v",
                                f, f.SegmentLocs, i, options, err)
                }

                a[i] = seg
        }

        f.ss = &segmentStack{
                options: &options.CollectionOptions,
                a:       a,
                refs:    1,
        }

        return mrefs, nil
}

// --------------------------------------------------------

// ChildCollectionNames returns an array of child collection name strings.
func (f *Footer) ChildCollectionNames() ([]string, error) {
        var childNames = make([]string, len(f.ChildFooters))
        idx := 0
        for name := range f.ChildFooters {
                childNames[idx] = name
                idx++
        }
        return childNames, nil
}

// ChildCollectionSnapshot returns a Snapshot on a given child
// collection by its name.
func (f *Footer) ChildCollectionSnapshot(childCollectionName string) (
        Snapshot, error) {
        childFooter, exists := f.ChildFooters[childCollectionName]
        if !exists {
                return nil, nil
        }
        childFooter.AddRef()
        return childFooter, nil
}

// Close decrements the ref count on this footer
func (f *Footer) Close() error {
        f.DecRef()
        return nil
}

// AddRef increases the ref count on this footer
func (f *Footer) AddRef() {
        f.m.Lock()
        f.refs++
        f.m.Unlock()
}

// DecRef decreases the ref count on this footer
func (f *Footer) DecRef() {
        f.m.Lock()
        f.refs--
        if f.refs <= 0 {
                f.SegmentLocs.DecRef()
                f.SegmentLocs = nil
                f.ss = nil
        }
        f.m.Unlock()
}

// Length returns the length of this footer
func (f *Footer) Length() uint64 {
        jBuf, err := json.Marshal(f)
        if err != nil {
                return 0
        }

        footerLen := footerBegLen + len(jBuf) + footerEndLen
        return uint64(footerLen)
}

// --------------------------------------------------------

// segmentLocs returns the current SegmentLocs and segmentStack for
// a footer, while also incrementing the ref-count on the footer.  The
// caller must DecRef() the footer when done.
func (f *Footer) segmentLocs() (SegmentLocs, *segmentStack) {
        f.m.Lock()

        f.refs++

        slocs, ss := f.SegmentLocs, f.ss

        f.m.Unlock()

        return slocs, ss
}

// --------------------------------------------------------

// Get retrieves a val from the footer, and will return nil val
// if the entry does not exist in the footer.
func (f *Footer) Get(key []byte, readOptions ReadOptions) ([]byte, error) {
        _, ss := f.segmentLocs()
        if ss == nil {
                f.DecRef()
                return nil, nil
        }

        rv, err := ss.Get(key, readOptions)
        if err == nil && rv != nil && !readOptions.NoCopyValue {
                rv = append(make([]byte, 0, len(rv)), rv...) // Copy.
        }

        f.DecRef()

        return rv, err
}

// StartIterator returns a new Iterator instance on this footer.
//
// On success, the returned Iterator will be positioned so that
// Iterator.Current() will either provide the first entry in the
// range or ErrIteratorDone.
//
// A startKeyIncl of nil means the logical "bottom-most" possible key
// and an endKeyExcl of nil means the logical "top-most" possible key.
func (f *Footer) StartIterator(startKeyIncl, endKeyExcl []byte,
        iteratorOptions IteratorOptions) (Iterator, error) {
        _, ss := f.segmentLocs()
        if ss == nil {
                f.DecRef()
                return nil, nil
        }

        iter, err := ss.StartIterator(startKeyIncl, endKeyExcl, iteratorOptions)
        if err != nil || iter == nil {
                f.DecRef()
                return nil, err
        }

        initCloser, ok := iter.(InitCloser)
        if !ok || initCloser == nil {
                iter.Close()
                f.DecRef()
                return nil, ErrUnexpected
        }

        err = initCloser.InitCloser(f)
        if err != nil {
                iter.Close()
                f.DecRef()
                return nil, err
        }

        return iter, nil
}

1	// Copyright (c) 2016 Couchbase, Inc.
2	// Licensed under the Apache License, Version 2.0 (the "License");
3	// you may not use this file except in compliance with the
4	// License. You may obtain a copy of the License at
5	// http://www.apache.org/licenses/LICENSE-2.0
6	// Unless required by applicable law or agreed to in writing,
7	// software distributed under the License is distributed on an "AS
8	// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
9	// express or implied. See the License for the specific language
10	// governing permissions and limitations under the License.
11
12	package moss
13
14	import (
15	"bytes"
16	"encoding/binary"
17	"encoding/json"
18	"fmt"
19	"time"
20
21	"github.com/edsrzf/mmap-go"
22	)
23
24	func (s Store) persistFooter(file File, footer Footer,
25	options StorePersistOptions) error {	187✔
26	startTime := time.Now()	187✔
27		187✔
28	if !options.NoSync {	373✔
29	err := file.Sync()	186✔
30	if err != nil {	186✔
31	return err	×
32	}	×
33	}
34
35	err := s.persistFooterUnsynced(file, footer)	187✔
36	if err != nil {	187✔
37	return err	×
38	}	×
39
40	if !options.NoSync {	373✔
41	err = file.Sync()	186✔
42	}	186✔
43
44	if err == nil {	374✔
45	s.histograms["PersistFooterUsecs"].Add(	187✔
46	uint64(time.Since(startTime).Nanoseconds()/1000), 1)	187✔
47	}	187✔
48
49	return err	187✔
50	}
51
52	func (s Store) persistFooterUnsynced(file File, footer Footer) error {	187✔
53	jBuf, err := json.Marshal(footer)	187✔
54	if err != nil {	187✔
55	return err	×
56	}	×
57
58	finfo, err := file.Stat()	187✔
59	if err != nil {	187✔
60	return err	×
61	}	×
62
63	footerPos := pageAlignCeil(finfo.Size())	187✔
64	footerLen := footerBegLen + len(jBuf) + footerEndLen	187✔
65		187✔
66	footerBuf := bytes.NewBuffer(make([]byte, 0, footerLen))	187✔
67	footerBuf.Write(StoreMagicBeg)	187✔
68	footerBuf.Write(StoreMagicBeg)	187✔
69	binary.Write(footerBuf, StoreEndian, uint32(StoreVersion))	187✔
70	binary.Write(footerBuf, StoreEndian, uint32(footerLen))	187✔
71	footerBuf.Write(jBuf)	187✔
72	binary.Write(footerBuf, StoreEndian, footerPos)	187✔
73	binary.Write(footerBuf, StoreEndian, uint32(footerLen))	187✔
74	footerBuf.Write(StoreMagicEnd)	187✔
75	footerBuf.Write(StoreMagicEnd)	187✔
76		187✔
77	footerWritten, err := file.WriteAt(footerBuf.Bytes(), footerPos)	187✔
78	if err != nil {	187✔
79	return err	×
80	}	×
81	if footerWritten != len(footerBuf.Bytes()) {	187✔
82	return fmt.Errorf("store: persistFooter error writing all footerBuf")	×
83	}	×
84
85	footer.fileName = finfo.Name()	187✔
86	footer.filePos = footerPos	187✔
87		187✔
88	return nil	187✔
89	}
90
91	// --------------------------------------------------------
92
93	// ReadFooter reads the last valid Footer from a file.
94	func ReadFooter(options StoreOptions, file File) (Footer, error) {	114✔
95	finfo, err := file.Stat()	114✔
96	if err != nil {	114✔
97	return nil, err	×
98	}	×
99
100	fref := &FileRef{file: file, refs: 1}	114✔
101		114✔
102	// To avoid an EOF while reading, start scanning the footer from	114✔
103	// the last byte. This is under the assumption that the footer is	114✔
104	// at least 2 bytes long.	114✔
105	f, err := ScanFooter(options, fref, finfo.Name(), finfo.Size()-1)	114✔
106	if err != nil {	118✔
107	return nil, err	4✔
108	}	4✔
109
110	fref.DecRef() // ScanFooter added its own ref-counts on success.	110✔
111		110✔
112	return f, err	110✔
113	}
114
115	// --------------------------------------------------------
116
117	// ScanFooter scans a file backwards from the given pos for a valid
118	// Footer, adding ref-counts to fref on success.
119	func ScanFooter(options StoreOptions, fref FileRef, fileName string,
120	pos int64) (*Footer, error) {	117✔
121	footerBeg := make([]byte, footerBegLen)	117✔
122		117✔
123	// Align pos to the start of a page (floor).	117✔
124	pos = pageAlignFloor(pos)	117✔
125		117✔
126	for {	234✔
127	for { // Scan for StoreMagicBeg, which may be a potential footer.	234✔
128	if pos <= 0 {	122✔
129	return nil, ErrNoValidFooter	5✔
130	}	5✔
131
132	n, err := fref.file.ReadAt(footerBeg, pos)	112✔
133	if err != nil {	112✔
134	return nil, err	×
135	}	×
136
137	if n == footerBegLen &&	112✔
138	bytes.Equal(StoreMagicBeg, footerBeg[:lenMagicBeg]) &&	112✔
139	bytes.Equal(StoreMagicBeg, footerBeg[lenMagicBeg:2*lenMagicBeg]) {	224✔
140	break	112✔
141	}
142
143	// Move pos back by page size.
144	pos -= int64(StorePageSize)	×
145	}
146
147	// Read and check the potential footer.
148	footerBegBuf := bytes.NewBuffer(footerBeg[2*lenMagicBeg:])	112✔
149		112✔
150	var version uint32	112✔
151	if err := binary.Read(footerBegBuf, StoreEndian, &version); err != nil {	112✔
152	return nil, err	×
153	}	×
154	if version != StoreVersion {	112✔
155	return nil, fmt.Errorf("store: version mismatch, "+	×
156	"current: %v != found: %v", StoreVersion, version)	×
157	}	×
158
159	var length uint32	112✔
160	if err := binary.Read(footerBegBuf, StoreEndian, &length); err != nil {	112✔
161	return nil, err	×
162	}	×
163
164	data := make([]byte, int64(length)-int64(footerBegLen))	112✔
165		112✔
166	n, err := fref.file.ReadAt(data, pos+int64(footerBegLen))	112✔
167	if err != nil {	112✔
168	return nil, err	×
169	}	×
170
171	if n == len(data) &&	112✔
172	bytes.Equal(StoreMagicEnd, data[n-lenMagicEnd*2:n-lenMagicEnd]) &&	112✔
173	bytes.Equal(StoreMagicEnd, data[n-lenMagicEnd:]) {	224✔
174		112✔
175	content := int(length) - footerBegLen - footerEndLen	112✔
176	b := bytes.NewBuffer(data[content:])	112✔
177		112✔
178	var offset int64	112✔
179	if err = binary.Read(b, StoreEndian, &offset); err != nil {	112✔
180	return nil, err	×
181	}	×
182	if offset != pos {	112✔
183	return nil, fmt.Errorf("store: offset mismatch, "+	×
184	"wanted: %v != found: %v", offset, pos)	×
185	}	×
186
187	var length1 uint32	112✔
188	if err = binary.Read(b, StoreEndian, &length1); err != nil {	112✔
189	return nil, err	×
190	}	×
191	if length1 != length {	112✔
192	return nil, fmt.Errorf("store: length mismatch, "+	×
193	"wanted: %v != found: %v", length1, length)	×
194	}	×
195
196	f := &Footer{refs: 1, fileName: fileName, filePos: offset}	112✔
197		112✔
198	err = json.Unmarshal(data[:content], f)	112✔
199	if err != nil {	112✔
200	return nil, err	×
201	}	×
202
203	// json.Unmarshal would have just loaded the map.
204	// We now need to load each segment into the map.
205	// Also recursively load child footer segment stacks.
206	err = f.loadSegments(options, fref)	112✔
207	if err != nil {	112✔
208	return nil, err	×
209	}	×
210
211	return f, nil	112✔
212	}
213	// Else, invalid footer - StoreMagicEnd missing and/or file
214	// pos out of bounds.
215
216	// Footer was invalid, so keep scanning.
217	pos -= int64(StorePageSize)	×
218	}
219	}
220
221	// --------------------------------------------------------
222
223	// loadSegments() loads the segments of a footer. Adds new ref-counts
224	// to the fref on success. The footer will be in an already closed
225	// state on error.
226	func (f Footer) loadSegments(options StoreOptions, fref *FileRef) (err error) {	204✔
227	// Track mrefs that we need to DecRef() if there's an error.	204✔
228	mrefs := make([]*mmapRef, 0, len(f.SegmentLocs))	204✔
229	mrefs, err = f.doLoadSegments(options, fref, mrefs)	204✔
230	if err != nil {	204✔
231	for _, mref := range mrefs {	×
232	mref.DecRef()	×
233	}	×
234	return err	×
235	}
236
237	return nil	204✔
238	}
239
240	func (f Footer) doLoadSegments(options StoreOptions, fref *FileRef,
241	mrefs []mmapRef) (mrefsSoFar []mmapRef, err error) {	257✔
242	// Recursively load the childFooters first.	257✔
243	for _, childFooter := range f.ChildFooters {	310✔
244	mrefs, err = childFooter.doLoadSegments(options, fref, mrefs)	53✔
245	if err != nil {	53✔
246	return mrefs, err	×
247	}	×
248	}
249
250	if f.ss != nil && f.ss.a != nil {	257✔
251	return mrefs, nil	×
252	}	×
253
254	osFile := ToOsFile(fref.file)	257✔
255	if osFile == nil {	257✔
256	return mrefs, fmt.Errorf("store: doLoadSegments convert to os.File error")	×
257	}	×
258
259	a := make([]Segment, len(f.SegmentLocs))	257✔
260		257✔
261	for i := range f.SegmentLocs {	933✔
262	sloc := &f.SegmentLocs[i]	676✔
263		676✔
264	mref := sloc.mref	676✔
265	if mref != nil {	1,082✔
266	if mref.fref != fref {	406✔
267	return mrefs, fmt.Errorf("store: doLoadSegments fref mismatch")	×
268	}	×
269
270	mref.AddRef()	406✔
271	} else {	270✔
272	// We persist kvs before buf, so KvsOffset < BufOffset.	270✔
273	begOffset := int64(sloc.KvsOffset)	270✔
274	endOffset := int64(sloc.BufOffset + sloc.BufBytes)	270✔
275		270✔
276	nbytes := int(endOffset - begOffset)	270✔
277		270✔
278	// Some platforms (windows) only support mmap()'ing at an	270✔
279	// allocation granularity that's != to a page size, so	270✔
280	// calculate the actual offset/nbytes to use.	270✔
281	begOffsetActual := pageOffset(begOffset, int64(AllocationGranularity))	270✔
282	begOffsetDelta := int(begOffset - begOffsetActual)	270✔
283	nbytesActual := nbytes + begOffsetDelta	270✔
284		270✔
285	mm, err := mmap.MapRegion(osFile, nbytesActual, mmap.RDONLY, 0, begOffsetActual)	270✔
286	if err != nil {	270✔
287	return mrefs, fmt.Errorf("store: doLoadSegments mmap.Map(), err: %v", err)	×
288	}	×
289
290	fref.AddRef() // New mref owns 1 fref ref-count.	270✔
291		270✔
292	buf := mm[begOffsetDelta : begOffsetDelta+nbytes]	270✔
293		270✔
294	sloc.mref = &mmapRef{fref: fref, mm: mm, buf: buf, refs: 1}	270✔
295		270✔
296	mref = sloc.mref	270✔
297	}
298
299	mrefs = append(mrefs, mref)	676✔
300		676✔
301	segmentLoader, exists := SegmentLoaders[sloc.Kind]	676✔
302	if !exists \|\| segmentLoader == nil {	676✔
303	return mrefs, fmt.Errorf("store: unknown SegmentLoc kind, sloc: %+v", sloc)	×
304	}	×
305
306	seg, err := segmentLoader(sloc)	676✔
307	if err != nil {	676✔
308	return mrefs, fmt.Errorf("store: segmentLoader failed, footer: %+v,"+	×
309	" f.SegmentLocs: %+v, i: %d, options: %v err: %+v",	×
310	f, f.SegmentLocs, i, options, err)	×
311	}	×
312
313	a[i] = seg	676✔
314	}
315
316	f.ss = &segmentStack{	257✔
317	options: &options.CollectionOptions,	257✔
318	a: a,	257✔
319	refs: 1,	257✔
320	}	257✔
321		257✔
322	return mrefs, nil	257✔
323	}
324
325	// --------------------------------------------------------
326
327	// ChildCollectionNames returns an array of child collection name strings.
328	func (f *Footer) ChildCollectionNames() ([]string, error) {	×
329	var childNames = make([]string, len(f.ChildFooters))	×
330	idx := 0	×
331	for name := range f.ChildFooters {	×
332	childNames[idx] = name	×
333	idx++	×
334	}	×
335	return childNames, nil	×
336	}
337
338	// ChildCollectionSnapshot returns a Snapshot on a given child
339	// collection by its name.
340	func (f *Footer) ChildCollectionSnapshot(childCollectionName string) (
341	Snapshot, error) {	264✔
342	childFooter, exists := f.ChildFooters[childCollectionName]	264✔
343	if !exists {	280✔
344	return nil, nil	16✔
345	}	16✔
346	childFooter.AddRef()	248✔
347	return childFooter, nil	248✔
348	}
349
350	// Close decrements the ref count on this footer
351	func (f *Footer) Close() error {	497✔
352	f.DecRef()	497✔
353	return nil	497✔
354	}	497✔
355
356	// AddRef increases the ref count on this footer
357	func (f *Footer) AddRef() {	616✔
358	f.m.Lock()	616✔
359	f.refs++	616✔
360	f.m.Unlock()	616✔
361	}	616✔
362
363	// DecRef decreases the ref count on this footer
364	func (f *Footer) DecRef() {	4,156✔
365	f.m.Lock()	4,156✔
366	f.refs--	4,156✔
367	if f.refs <= 0 {	4,385✔
368	f.SegmentLocs.DecRef()	229✔
369	f.SegmentLocs = nil	229✔
370	f.ss = nil	229✔
371	}	229✔
372	f.m.Unlock()	4,156✔
373	}
374
375	// Length returns the length of this footer
376	func (f *Footer) Length() uint64 {	×
377	jBuf, err := json.Marshal(f)	×
378	if err != nil {	×
379	return 0	×
380	}	×
381
382	footerLen := footerBegLen + len(jBuf) + footerEndLen	×
383	return uint64(footerLen)	×
384	}
385
386	// --------------------------------------------------------
387
388	// segmentLocs returns the current SegmentLocs and segmentStack for
389	// a footer, while also incrementing the ref-count on the footer. The
390	// caller must DecRef() the footer when done.
391	func (f Footer) segmentLocs() (SegmentLocs, segmentStack) {	3,567✔
392	f.m.Lock()	3,567✔
393		3,567✔
394	f.refs++	3,567✔
395		3,567✔
396	slocs, ss := f.SegmentLocs, f.ss	3,567✔
397		3,567✔
398	f.m.Unlock()	3,567✔
399		3,567✔
400	return slocs, ss	3,567✔
401	}	3,567✔
402
403	// --------------------------------------------------------
404
405	// Get retrieves a val from the footer, and will return nil val
406	// if the entry does not exist in the footer.
407	func (f *Footer) Get(key []byte, readOptions ReadOptions) ([]byte, error) {	3,024✔
408	_, ss := f.segmentLocs()	3,024✔
409	if ss == nil {	3,024✔
410	f.DecRef()	×
411	return nil, nil	×
412	}	×
413
414	rv, err := ss.Get(key, readOptions)	3,024✔
415	if err == nil && rv != nil && !readOptions.NoCopyValue {	5,375✔
416	rv = append(make([]byte, 0, len(rv)), rv...) // Copy.	2,351✔
417	}	2,351✔
418
419	f.DecRef()	3,024✔
420		3,024✔
421	return rv, err	3,024✔
422	}
423
424	// StartIterator returns a new Iterator instance on this footer.
425	//
426	// On success, the returned Iterator will be positioned so that
427	// Iterator.Current() will either provide the first entry in the
428	// range or ErrIteratorDone.
429	//
430	// A startKeyIncl of nil means the logical "bottom-most" possible key
431	// and an endKeyExcl of nil means the logical "top-most" possible key.
432	func (f *Footer) StartIterator(startKeyIncl, endKeyExcl []byte,
433	iteratorOptions IteratorOptions) (Iterator, error) {	224✔
434	_, ss := f.segmentLocs()	224✔
435	if ss == nil {	224✔
436	f.DecRef()	×
437	return nil, nil	×
438	}	×
439
440	iter, err := ss.StartIterator(startKeyIncl, endKeyExcl, iteratorOptions)	224✔
441	if err != nil \|\| iter == nil {	224✔
442	f.DecRef()	×
443	return nil, err	×
444	}	×
445
446	initCloser, ok := iter.(InitCloser)	224✔
447	if !ok \|\| initCloser == nil {	224✔
448	iter.Close()	×
449	f.DecRef()	×
450	return nil, ErrUnexpected	×
451	}	×
452
453	err = initCloser.InitCloser(f)	224✔
454	if err != nil {	224✔
455	iter.Close()	×
456	f.DecRef()	×
457	return nil, err	×
458	}	×
459
460	return iter, nil	224✔
461	}

couchbase / moss / 254

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous