couchbase / moss / 306

pending completion

Build # 306

Build Type

push

travis-ci

Committed by

abhinavdangeti

Commit Message

<a href="https://github.com/couchbase/moss/commit/<a class=hub.com/couchbase/moss/commit/<a class="double-link" href="https://git"><a class=hub.com/couchbase/moss/commit/<a class="double-link" href="https://git"><a class=hub.com/couchbase/moss/commit/<a class="double-link" href="https://git"><a class=hub.com/couchbase/moss/commit/956632ec1bc3e28276d00ee2f22c3202f06efb12">956632ec1">&lt;a href=&quot;https://github.com/couchbase/moss/commit/</a><a class="double-link" href="https://github.com/couchbase/moss/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/couchbase/moss/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/couchbase/moss/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/couchbase/moss/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/couchbase/moss/commit/956632ec1bc3e28276d00ee2f22c3202f06efb12">956632ec1</a><a href="https://github.com/couchbase/moss/commit/956632ec1bc3e28276d00ee2f22c3202f06efb12">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/couchbase/moss/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/couchbase/moss/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/couchbase/moss/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/couchbase/moss/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/couchbase/moss/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/couchbase/moss/commit/956632ec1bc3e28276d00ee2f22c3202f06efb12&quot;&gt;956632ec1&lt;/a&gt;&lt;a href=&quot;https://github.com/couchbase/moss/commit/956632ec1bc3e28276d00ee2f22c3202f06efb12&quot;&gt;&amp;quot;&amp;gt;&amp;amp;lt;a href=&amp;amp;quot;https://github.com/couchbase/moss/commit/&amp;lt;/a&amp;gt;&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/couchbase/moss/commit/&amp;amp;lt;a class=&amp;amp;quot;double-link&amp;amp;quot; href=... (continued)

Run Details

7 of 7 new or added lines in 1 file covered. (100.0%)

3365 of 4055 relevant lines covered (82.98%)

6404183.53 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

66.76

/store_footer.go

//  Copyright (c) 2016 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package moss

import (
        "bytes"
        "encoding/binary"
        "encoding/json"
        "fmt"
        "time"

        "github.com/edsrzf/mmap-go"
)

func (s *Store) persistFooter(file File, footer *Footer,
        options StorePersistOptions) error {
        startTime := time.Now()

        if !options.NoSync {
                err := file.Sync()
                if err != nil {
                        return err
                }
        }

        err := s.persistFooterUnsynced(file, footer)
        if err != nil {
                return err
        }

        if !options.NoSync {
                err = file.Sync()
        }

        if err == nil {
                s.histograms["PersistFooterUsecs"].Add(
                        uint64(time.Since(startTime).Nanoseconds()/1000), 1)
        }

        return err
}

func (s *Store) persistFooterUnsynced(file File, footer *Footer) error {
        jBuf, err := json.Marshal(footer)
        if err != nil {
                return err
        }

        finfo, err := file.Stat()
        if err != nil {
                return err
        }

        footerPos := pageAlignCeil(finfo.Size())
        footerLen := footerBegLen + len(jBuf) + footerEndLen

        footerBuf := bytes.NewBuffer(make([]byte, 0, footerLen))
        footerBuf.Write(StoreMagicBeg)
        footerBuf.Write(StoreMagicBeg)
        binary.Write(footerBuf, StoreEndian, uint32(StoreVersion))
        binary.Write(footerBuf, StoreEndian, uint32(footerLen))
        footerBuf.Write(jBuf)
        binary.Write(footerBuf, StoreEndian, footerPos)
        binary.Write(footerBuf, StoreEndian, uint32(footerLen))
        footerBuf.Write(StoreMagicEnd)
        footerBuf.Write(StoreMagicEnd)

        footerWritten, err := file.WriteAt(footerBuf.Bytes(), footerPos)
        if err != nil {
                return err
        }
        if footerWritten != len(footerBuf.Bytes()) {
                return fmt.Errorf("store: persistFooter error writing all footerBuf")
        }

        if AllocationGranularity != StorePageSize {
                // Some platforms (windows) only support mmap()'ing at an
                // allocation granularity that's != to a page size.
                //
                // However if on such platforms there are empty segments, then
                // due to the extra space imposed by the above granularity
                // requirement, mmap() can fail complaining about insufficient
                // file space.
                //
                // To avoid this error, simply pad up the file up to a page
                // boundary.  This pad of zeroes will not interfere with file
                // recovery.
                padding := make([]byte, int(pageAlignCeil(int64(footerWritten)))-footerWritten)
                _, err = file.WriteAt(padding, footerPos+int64(footerWritten))
                if err != nil {
                        return err
                }
        }

        footer.fileName = finfo.Name()
        footer.filePos = footerPos

        return nil
}

// --------------------------------------------------------

// ReadFooter reads the last valid Footer from a file.
func ReadFooter(options *StoreOptions, file File) (*Footer, error) {
        finfo, err := file.Stat()
        if err != nil {
                return nil, err
        }

        fref := &FileRef{file: file, refs: 1}

        // To avoid an EOF while reading, start scanning the footer from
        // the last byte. This is under the assumption that the footer is
        // at least 2 bytes long.
        f, err := ScanFooter(options, fref, finfo.Name(), finfo.Size()-1)
        if err != nil {
                return nil, err
        }

        fref.DecRef() // ScanFooter added its own ref-counts on success.

        return f, err
}

// --------------------------------------------------------

// ScanFooter scans a file backwards from the given pos for a valid
// Footer, adding ref-counts to fref on success.
func ScanFooter(options *StoreOptions, fref *FileRef, fileName string,
        pos int64) (*Footer, error) {
        footerBeg := make([]byte, footerBegLen)

        // Align pos to the start of a page (floor).
        pos = pageAlignFloor(pos)

        for {
                for { // Scan for StoreMagicBeg, which may be a potential footer.
                        if pos <= 0 {
                                return nil, ErrNoValidFooter
                        }

                        n, err := fref.file.ReadAt(footerBeg, pos)
                        if err != nil {
                                return nil, err
                        }

                        if n == footerBegLen &&
                                bytes.Equal(StoreMagicBeg, footerBeg[:lenMagicBeg]) &&
                                bytes.Equal(StoreMagicBeg, footerBeg[lenMagicBeg:2*lenMagicBeg]) {
                                break
                        }

                        // Move pos back by page size.
                        pos -= int64(StorePageSize)
                }

                // Read and check the potential footer.
                footerBegBuf := bytes.NewBuffer(footerBeg[2*lenMagicBeg:])

                var version uint32
                if err := binary.Read(footerBegBuf, StoreEndian, &version); err != nil {
                        return nil, err
                }
                if version != StoreVersion {
                        return nil, fmt.Errorf("store: version mismatch, "+
                                "current: %v != found: %v", StoreVersion, version)
                }

                var length uint32
                if err := binary.Read(footerBegBuf, StoreEndian, &length); err != nil {
                        return nil, err
                }

                data := make([]byte, int64(length)-int64(footerBegLen))

                n, err := fref.file.ReadAt(data, pos+int64(footerBegLen))
                if err != nil {
                        return nil, err
                }

                if n == len(data) &&
                        bytes.Equal(StoreMagicEnd, data[n-lenMagicEnd*2:n-lenMagicEnd]) &&
                        bytes.Equal(StoreMagicEnd, data[n-lenMagicEnd:]) {

                        content := int(length) - footerBegLen - footerEndLen
                        b := bytes.NewBuffer(data[content:])

                        var offset int64
                        if err = binary.Read(b, StoreEndian, &offset); err != nil {
                                return nil, err
                        }
                        if offset != pos {
                                return nil, fmt.Errorf("store: offset mismatch, "+
                                        "wanted: %v != found: %v", offset, pos)
                        }

                        var length1 uint32
                        if err = binary.Read(b, StoreEndian, &length1); err != nil {
                                return nil, err
                        }
                        if length1 != length {
                                return nil, fmt.Errorf("store: length mismatch, "+
                                        "wanted: %v != found: %v", length1, length)
                        }

                        f := &Footer{refs: 1, fileName: fileName, filePos: offset}

                        err = json.Unmarshal(data[:content], f)
                        if err != nil {
                                return nil, err
                        }

                        // json.Unmarshal would have just loaded the map.
                        // We now need to load each segment into the map.
                        // Also recursively load child footer segment stacks.
                        err = f.loadSegments(options, fref)
                        if err != nil {
                                return nil, err
                        }

                        return f, nil
                }
                // Else, invalid footer - StoreMagicEnd missing and/or file
                // pos out of bounds.

                // Footer was invalid, so keep scanning.
                pos -= int64(StorePageSize)
        }
}

// --------------------------------------------------------

// loadSegments() loads the segments of a footer.  Adds new ref-counts
// to the fref on success.  The footer will be in an already closed
// state on error.
func (f *Footer) loadSegments(options *StoreOptions, fref *FileRef) (err error) {
        // Track mrefs that we need to DecRef() if there's an error.
        mrefs := make([]*mmapRef, 0, len(f.SegmentLocs))
        mrefs, err = f.doLoadSegments(options, fref, mrefs)
        if err != nil {
                for _, mref := range mrefs {
                        mref.DecRef()
                }
                return err
        }

        return nil
}

func (f *Footer) doLoadSegments(options *StoreOptions, fref *FileRef,
        mrefs []*mmapRef) (mrefsSoFar []*mmapRef, err error) {
        // Recursively load the childFooters first.
        for _, childFooter := range f.ChildFooters {
                mrefs, err = childFooter.doLoadSegments(options, fref, mrefs)
                if err != nil {
                        return mrefs, err
                }
        }

        if f.ss != nil && f.ss.a != nil {
                return mrefs, nil
        }

        osFile := ToOsFile(fref.file)
        if osFile == nil {
                return mrefs, fmt.Errorf("store: doLoadSegments convert to os.File error")
        }

        a := make([]Segment, len(f.SegmentLocs))

        for i := range f.SegmentLocs {
                sloc := &f.SegmentLocs[i]

                mref := sloc.mref
                if mref != nil {
                        if mref.fref != fref {
                                return mrefs, fmt.Errorf("store: doLoadSegments fref mismatch")
                        }

                        mref.AddRef()
                        mrefs = append(mrefs, mref)
                } else {
                        // We persist kvs before buf, so KvsOffset < BufOffset.
                        begOffset := int64(sloc.KvsOffset)
                        endOffset := int64(sloc.BufOffset + sloc.BufBytes)

                        nbytes := int(endOffset - begOffset)

                        // Some platforms (windows) only support mmap()'ing at an
                        // allocation granularity that's != to a page size, so
                        // calculate the actual offset/nbytes to use.
                        begOffsetActual := pageOffset(begOffset, int64(AllocationGranularity))
                        begOffsetDelta := int(begOffset - begOffsetActual)
                        nbytesActual := nbytes + begOffsetDelta

                        // check whether the actual file fits within the footer offsets
                        fstats, err := osFile.Stat()
                        if err != nil || nbytesActual > int(fstats.Size()) {
                                return mrefs, fmt.Errorf("store: doLoadSegments corrupted "+
                                        "file: %s, err: %+v", fstats.Name(), err)
                        }

                        mm, err := mmap.MapRegion(osFile, nbytesActual, mmap.RDONLY, 0, begOffsetActual)
                        if err != nil {
                                return mrefs,
                                        fmt.Errorf("store: doLoadSegments mmap.Map(),"+
                                                " begOffsetActual = %v, nbytesActual = %v, sloc = %+v,"+
                                                " file: %s, file mode: %v, file modification time: %v,"+
                                                " footer: %+v, f.SegmentLocs: %+v, err: %v",
                                                begOffsetActual, nbytesActual, sloc,
                                                fstats.Name(), fstats.Mode(), fstats.ModTime(),
                                                f, f.SegmentLocs, err)
                        }

                        fref.AddRef() // New mref owns 1 fref ref-count.

                        buf := mm[begOffsetDelta : begOffsetDelta+nbytes]

                        sloc.mref = &mmapRef{fref: fref, mm: mm, buf: buf, refs: 1}

                        mref = sloc.mref
                        mrefs = append(mrefs, mref)

                        segmentLoader, exists := SegmentLoaders[sloc.Kind]
                        if !exists || segmentLoader == nil {
                                return mrefs, fmt.Errorf("store: unknown SegmentLoc kind, sloc: %+v", sloc)
                        }

                        seg, err := segmentLoader(sloc)
                        if err != nil {
                                return mrefs, fmt.Errorf("store: segmentLoader failed, footer: %+v,"+
                                        " f.SegmentLocs: %+v, i: %d, options: %v err: %+v",
                                        f, f.SegmentLocs, i, options, err)
                        }

                        segmentKeysIndexMaxBytes := options.SegmentKeysIndexMaxBytes
                        if segmentKeysIndexMaxBytes == 0 {
                                segmentKeysIndexMaxBytes = DefaultStoreOptions.SegmentKeysIndexMaxBytes
                        }

                        segmentKeysIndexMinKeyBytes := options.SegmentKeysIndexMinKeyBytes
                        if segmentKeysIndexMinKeyBytes == 0 {
                                segmentKeysIndexMinKeyBytes = DefaultStoreOptions.SegmentKeysIndexMinKeyBytes
                        }

                        if segmentKeysIndexMaxBytes > 0 {
                                if a, ok := seg.(*segment); ok {
                                        a.buildIndex(segmentKeysIndexMaxBytes, segmentKeysIndexMinKeyBytes)
                                }
                        }

                        mref.SetExt(seg)
                }

                a[i] = mref.GetExt().(Segment)
        }

        f.ss = &segmentStack{
                options: &options.CollectionOptions,
                a:       a,
                refs:    1,
        }

        return mrefs, nil
}

// --------------------------------------------------------

// ChildCollectionNames returns an array of child collection name strings.
func (f *Footer) ChildCollectionNames() ([]string, error) {
        var childNames = make([]string, len(f.ChildFooters))
        idx := 0
        for name := range f.ChildFooters {
                childNames[idx] = name
                idx++
        }
        return childNames, nil
}

// ChildCollectionSnapshot returns a Snapshot on a given child
// collection by its name.
func (f *Footer) ChildCollectionSnapshot(childCollectionName string) (
        Snapshot, error) {
        childFooter, exists := f.ChildFooters[childCollectionName]
        if !exists {
                return nil, nil
        }
        childFooter.AddRef()
        return childFooter, nil
}

// Close decrements the ref count on this footer
func (f *Footer) Close() error {
        f.DecRef()
        return nil
}

// AddRef increases the ref count on this footer
func (f *Footer) AddRef() {
        f.m.Lock()
        f.refs++
        f.m.Unlock()
}

// DecRef decreases the ref count on this footer
func (f *Footer) DecRef() {
        f.m.Lock()
        f.refs--
        if f.refs <= 0 {
                f.SegmentLocs.DecRef()
                f.SegmentLocs = nil
                f.ss = nil
        }
        f.m.Unlock()
}

// Length returns the length of this footer
func (f *Footer) Length() uint64 {
        jBuf, err := json.Marshal(f)
        if err != nil {
                return 0
        }

        footerLen := footerBegLen + len(jBuf) + footerEndLen
        return uint64(footerLen)
}

// --------------------------------------------------------

// segmentLocs returns the current SegmentLocs and segmentStack for
// a footer, while also incrementing the ref-count on the footer.  The
// caller must DecRef() the footer when done.
func (f *Footer) segmentLocs() (SegmentLocs, *segmentStack) {
        f.m.Lock()

        f.refs++

        slocs, ss := f.SegmentLocs, f.ss

        f.m.Unlock()

        return slocs, ss
}

// --------------------------------------------------------

// Get retrieves a val from the footer, and will return nil val
// if the entry does not exist in the footer.
func (f *Footer) Get(key []byte, readOptions ReadOptions) ([]byte, error) {
        _, ss := f.segmentLocs()
        if ss == nil {
                f.DecRef()
                return nil, nil
        }

        rv, err := ss.Get(key, readOptions)
        if err == nil && rv != nil && !readOptions.NoCopyValue {
                rv = append(make([]byte, 0, len(rv)), rv...) // Copy.
        }

        f.DecRef()

        return rv, err
}

// StartIterator returns a new Iterator instance on this footer.
//
// On success, the returned Iterator will be positioned so that
// Iterator.Current() will either provide the first entry in the
// range or ErrIteratorDone.
//
// A startKeyIncl of nil means the logical "bottom-most" possible key
// and an endKeyExcl of nil means the logical "top-most" possible key.
func (f *Footer) StartIterator(startKeyIncl, endKeyExcl []byte,
        iteratorOptions IteratorOptions) (Iterator, error) {
        _, ss := f.segmentLocs()
        if ss == nil {
                f.DecRef()
                return nil, nil
        }

        iter, err := ss.StartIterator(startKeyIncl, endKeyExcl, iteratorOptions)
        if err != nil || iter == nil {
                f.DecRef()
                return nil, err
        }

        initCloser, ok := iter.(InitCloser)
        if !ok || initCloser == nil {
                iter.Close()
                f.DecRef()
                return nil, ErrUnexpected
        }

        err = initCloser.InitCloser(f)
        if err != nil {
                iter.Close()
                f.DecRef()
                return nil, err
        }

        return iter, nil
}

1	// Copyright (c) 2016 Couchbase, Inc.
2	// Licensed under the Apache License, Version 2.0 (the "License");
3	// you may not use this file except in compliance with the
4	// License. You may obtain a copy of the License at
5	// http://www.apache.org/licenses/LICENSE-2.0
6	// Unless required by applicable law or agreed to in writing,
7	// software distributed under the License is distributed on an "AS
8	// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
9	// express or implied. See the License for the specific language
10	// governing permissions and limitations under the License.
11
12	package moss
13
14	import (
15	"bytes"
16	"encoding/binary"
17	"encoding/json"
18	"fmt"
19	"time"
20
21	"github.com/edsrzf/mmap-go"
22	)
23
24	func (s Store) persistFooter(file File, footer Footer,
25	options StorePersistOptions) error {	1,465✔
26	startTime := time.Now()	1,465✔
27		1,465✔
28	if !options.NoSync {	2,929✔
29	err := file.Sync()	1,464✔
30	if err != nil {	1,464✔
31	return err	×
32	}	×
33	}
34
35	err := s.persistFooterUnsynced(file, footer)	1,465✔
36	if err != nil {	1,465✔
37	return err	×
38	}	×
39
40	if !options.NoSync {	2,929✔
41	err = file.Sync()	1,464✔
42	}	1,464✔
43
44	if err == nil {	2,930✔
45	s.histograms["PersistFooterUsecs"].Add(	1,465✔
46	uint64(time.Since(startTime).Nanoseconds()/1000), 1)	1,465✔
47	}	1,465✔
48
49	return err	1,465✔
50	}
51
52	func (s Store) persistFooterUnsynced(file File, footer Footer) error {	1,465✔
53	jBuf, err := json.Marshal(footer)	1,465✔
54	if err != nil {	1,465✔
55	return err	×
56	}	×
57
58	finfo, err := file.Stat()	1,465✔
59	if err != nil {	1,465✔
60	return err	×
61	}	×
62
63	footerPos := pageAlignCeil(finfo.Size())	1,465✔
64	footerLen := footerBegLen + len(jBuf) + footerEndLen	1,465✔
65		1,465✔
66	footerBuf := bytes.NewBuffer(make([]byte, 0, footerLen))	1,465✔
67	footerBuf.Write(StoreMagicBeg)	1,465✔
68	footerBuf.Write(StoreMagicBeg)	1,465✔
69	binary.Write(footerBuf, StoreEndian, uint32(StoreVersion))	1,465✔
70	binary.Write(footerBuf, StoreEndian, uint32(footerLen))	1,465✔
71	footerBuf.Write(jBuf)	1,465✔
72	binary.Write(footerBuf, StoreEndian, footerPos)	1,465✔
73	binary.Write(footerBuf, StoreEndian, uint32(footerLen))	1,465✔
74	footerBuf.Write(StoreMagicEnd)	1,465✔
75	footerBuf.Write(StoreMagicEnd)	1,465✔
76		1,465✔
77	footerWritten, err := file.WriteAt(footerBuf.Bytes(), footerPos)	1,465✔
78	if err != nil {	1,465✔
79	return err	×
80	}	×
81	if footerWritten != len(footerBuf.Bytes()) {	1,465✔
82	return fmt.Errorf("store: persistFooter error writing all footerBuf")	×
83	}	×
84
85	if AllocationGranularity != StorePageSize {	1,465✔
86	// Some platforms (windows) only support mmap()'ing at an	×
87	// allocation granularity that's != to a page size.	×
88	//	×
89	// However if on such platforms there are empty segments, then	×
90	// due to the extra space imposed by the above granularity	×
91	// requirement, mmap() can fail complaining about insufficient	×
92	// file space.	×
93	//	×
94	// To avoid this error, simply pad up the file up to a page	×
95	// boundary. This pad of zeroes will not interfere with file	×
96	// recovery.	×
97	padding := make([]byte, int(pageAlignCeil(int64(footerWritten)))-footerWritten)	×
98	_, err = file.WriteAt(padding, footerPos+int64(footerWritten))	×
99	if err != nil {	×
100	return err	×
101	}	×
102	}
103
104	footer.fileName = finfo.Name()	1,465✔
105	footer.filePos = footerPos	1,465✔
106		1,465✔
107	return nil	1,465✔
108	}
109
110	// --------------------------------------------------------
111
112	// ReadFooter reads the last valid Footer from a file.
113	func ReadFooter(options StoreOptions, file File) (Footer, error) {	21✔
114	finfo, err := file.Stat()	21✔
115	if err != nil {	21✔
116	return nil, err	×
117	}	×
118
119	fref := &FileRef{file: file, refs: 1}	21✔
120		21✔
121	// To avoid an EOF while reading, start scanning the footer from	21✔
122	// the last byte. This is under the assumption that the footer is	21✔
123	// at least 2 bytes long.	21✔
124	f, err := ScanFooter(options, fref, finfo.Name(), finfo.Size()-1)	21✔
125	if err != nil {	25✔
126	return nil, err	4✔
127	}	4✔
128
129	fref.DecRef() // ScanFooter added its own ref-counts on success.	17✔
130		17✔
131	return f, err	17✔
132	}
133
134	// --------------------------------------------------------
135
136	// ScanFooter scans a file backwards from the given pos for a valid
137	// Footer, adding ref-counts to fref on success.
138	func ScanFooter(options StoreOptions, fref FileRef, fileName string,
139	pos int64) (*Footer, error) {	24✔
140	footerBeg := make([]byte, footerBegLen)	24✔
141		24✔
142	// Align pos to the start of a page (floor).	24✔
143	pos = pageAlignFloor(pos)	24✔
144		24✔
145	for {	48✔
146	for { // Scan for StoreMagicBeg, which may be a potential footer.	48✔
147	if pos <= 0 {	29✔
148	return nil, ErrNoValidFooter	5✔
149	}	5✔
150
151	n, err := fref.file.ReadAt(footerBeg, pos)	19✔
152	if err != nil {	19✔
153	return nil, err	×
154	}	×
155
156	if n == footerBegLen &&	19✔
157	bytes.Equal(StoreMagicBeg, footerBeg[:lenMagicBeg]) &&	19✔
158	bytes.Equal(StoreMagicBeg, footerBeg[lenMagicBeg:2*lenMagicBeg]) {	38✔
159	break	19✔
160	}
161
162	// Move pos back by page size.
163	pos -= int64(StorePageSize)	×
164	}
165
166	// Read and check the potential footer.
167	footerBegBuf := bytes.NewBuffer(footerBeg[2*lenMagicBeg:])	19✔
168		19✔
169	var version uint32	19✔
170	if err := binary.Read(footerBegBuf, StoreEndian, &version); err != nil {	19✔
171	return nil, err	×
172	}	×
173	if version != StoreVersion {	19✔
174	return nil, fmt.Errorf("store: version mismatch, "+	×
175	"current: %v != found: %v", StoreVersion, version)	×
176	}	×
177
178	var length uint32	19✔
179	if err := binary.Read(footerBegBuf, StoreEndian, &length); err != nil {	19✔
180	return nil, err	×
181	}	×
182
183	data := make([]byte, int64(length)-int64(footerBegLen))	19✔
184		19✔
185	n, err := fref.file.ReadAt(data, pos+int64(footerBegLen))	19✔
186	if err != nil {	19✔
187	return nil, err	×
188	}	×
189
190	if n == len(data) &&	19✔
191	bytes.Equal(StoreMagicEnd, data[n-lenMagicEnd*2:n-lenMagicEnd]) &&	19✔
192	bytes.Equal(StoreMagicEnd, data[n-lenMagicEnd:]) {	38✔
193		19✔
194	content := int(length) - footerBegLen - footerEndLen	19✔
195	b := bytes.NewBuffer(data[content:])	19✔
196		19✔
197	var offset int64	19✔
198	if err = binary.Read(b, StoreEndian, &offset); err != nil {	19✔
199	return nil, err	×
200	}	×
201	if offset != pos {	19✔
202	return nil, fmt.Errorf("store: offset mismatch, "+	×
203	"wanted: %v != found: %v", offset, pos)	×
204	}	×
205
206	var length1 uint32	19✔
207	if err = binary.Read(b, StoreEndian, &length1); err != nil {	19✔
208	return nil, err	×
209	}	×
210	if length1 != length {	19✔
211	return nil, fmt.Errorf("store: length mismatch, "+	×
212	"wanted: %v != found: %v", length1, length)	×
213	}	×
214
215	f := &Footer{refs: 1, fileName: fileName, filePos: offset}	19✔
216		19✔
217	err = json.Unmarshal(data[:content], f)	19✔
218	if err != nil {	19✔
219	return nil, err	×
220	}	×
221
222	// json.Unmarshal would have just loaded the map.
223	// We now need to load each segment into the map.
224	// Also recursively load child footer segment stacks.
225	err = f.loadSegments(options, fref)	19✔
226	if err != nil {	19✔
227	return nil, err	×
228	}	×
229
230	return f, nil	19✔
231	}
232	// Else, invalid footer - StoreMagicEnd missing and/or file
233	// pos out of bounds.
234
235	// Footer was invalid, so keep scanning.
236	pos -= int64(StorePageSize)	×
237	}
238	}
239
240	// --------------------------------------------------------
241
242	// loadSegments() loads the segments of a footer. Adds new ref-counts
243	// to the fref on success. The footer will be in an already closed
244	// state on error.
245	func (f Footer) loadSegments(options StoreOptions, fref *FileRef) (err error) {	1,483✔
246	// Track mrefs that we need to DecRef() if there's an error.	1,483✔
247	mrefs := make([]*mmapRef, 0, len(f.SegmentLocs))	1,483✔
248	mrefs, err = f.doLoadSegments(options, fref, mrefs)	1,483✔
249	if err != nil {	1,483✔
250	for _, mref := range mrefs {	×
251	mref.DecRef()	×
252	}	×
253	return err	×
254	}
255
256	return nil	1,483✔
257	}
258
259	func (f Footer) doLoadSegments(options StoreOptions, fref *FileRef,
260	mrefs []mmapRef) (mrefsSoFar []mmapRef, err error) {	1,521✔
261	// Recursively load the childFooters first.	1,521✔
262	for _, childFooter := range f.ChildFooters {	1,559✔
263	mrefs, err = childFooter.doLoadSegments(options, fref, mrefs)	38✔
264	if err != nil {	38✔
265	return mrefs, err	×
266	}	×
267	}
268
269	if f.ss != nil && f.ss.a != nil {	1,521✔
270	return mrefs, nil	×
271	}	×
272
273	osFile := ToOsFile(fref.file)	1,521✔
274	if osFile == nil {	1,521✔
275	return mrefs, fmt.Errorf("store: doLoadSegments convert to os.File error")	×
276	}	×
277
278	a := make([]Segment, len(f.SegmentLocs))	1,521✔
279		1,521✔
280	for i := range f.SegmentLocs {	8,835✔
281	sloc := &f.SegmentLocs[i]	7,314✔
282		7,314✔
283	mref := sloc.mref	7,314✔
284	if mref != nil {	12,784✔
285	if mref.fref != fref {	5,470✔
286	return mrefs, fmt.Errorf("store: doLoadSegments fref mismatch")	×
287	}	×
288
289	mref.AddRef()	5,470✔
290	mrefs = append(mrefs, mref)	5,470✔
291	} else {	1,844✔
292	// We persist kvs before buf, so KvsOffset < BufOffset.	1,844✔
293	begOffset := int64(sloc.KvsOffset)	1,844✔
294	endOffset := int64(sloc.BufOffset + sloc.BufBytes)	1,844✔
295		1,844✔
296	nbytes := int(endOffset - begOffset)	1,844✔
297		1,844✔
298	// Some platforms (windows) only support mmap()'ing at an	1,844✔
299	// allocation granularity that's != to a page size, so	1,844✔
300	// calculate the actual offset/nbytes to use.	1,844✔
301	begOffsetActual := pageOffset(begOffset, int64(AllocationGranularity))	1,844✔
302	begOffsetDelta := int(begOffset - begOffsetActual)	1,844✔
303	nbytesActual := nbytes + begOffsetDelta	1,844✔
304		1,844✔
305	// check whether the actual file fits within the footer offsets	1,844✔
306	fstats, err := osFile.Stat()	1,844✔
307	if err != nil \|\| nbytesActual > int(fstats.Size()) {	1,844✔
308	return mrefs, fmt.Errorf("store: doLoadSegments corrupted "+	×
309	"file: %s, err: %+v", fstats.Name(), err)	×
310	}	×
311
312	mm, err := mmap.MapRegion(osFile, nbytesActual, mmap.RDONLY, 0, begOffsetActual)	1,844✔
313	if err != nil {	1,844✔
314	return mrefs,	×
315	fmt.Errorf("store: doLoadSegments mmap.Map(),"+	×
316	" begOffsetActual = %v, nbytesActual = %v, sloc = %+v,"+	×
317	" file: %s, file mode: %v, file modification time: %v,"+	×
318	" footer: %+v, f.SegmentLocs: %+v, err: %v",	×
319	begOffsetActual, nbytesActual, sloc,	×
320	fstats.Name(), fstats.Mode(), fstats.ModTime(),	×
321	f, f.SegmentLocs, err)	×
322	}	×
323
324	fref.AddRef() // New mref owns 1 fref ref-count.	1,844✔
325		1,844✔
326	buf := mm[begOffsetDelta : begOffsetDelta+nbytes]	1,844✔
327		1,844✔
328	sloc.mref = &mmapRef{fref: fref, mm: mm, buf: buf, refs: 1}	1,844✔
329		1,844✔
330	mref = sloc.mref	1,844✔
331	mrefs = append(mrefs, mref)	1,844✔
332		1,844✔
333	segmentLoader, exists := SegmentLoaders[sloc.Kind]	1,844✔
334	if !exists \|\| segmentLoader == nil {	1,844✔
335	return mrefs, fmt.Errorf("store: unknown SegmentLoc kind, sloc: %+v", sloc)	×
336	}	×
337
338	seg, err := segmentLoader(sloc)	1,844✔
339	if err != nil {	1,844✔
340	return mrefs, fmt.Errorf("store: segmentLoader failed, footer: %+v,"+	×
341	" f.SegmentLocs: %+v, i: %d, options: %v err: %+v",	×
342	f, f.SegmentLocs, i, options, err)	×
343	}	×
344
345	segmentKeysIndexMaxBytes := options.SegmentKeysIndexMaxBytes	1,844✔
346	if segmentKeysIndexMaxBytes == 0 {	2,512✔
347	segmentKeysIndexMaxBytes = DefaultStoreOptions.SegmentKeysIndexMaxBytes	668✔
348	}	668✔
349
350	segmentKeysIndexMinKeyBytes := options.SegmentKeysIndexMinKeyBytes	1,844✔
351	if segmentKeysIndexMinKeyBytes == 0 {	2,512✔
352	segmentKeysIndexMinKeyBytes = DefaultStoreOptions.SegmentKeysIndexMinKeyBytes	668✔
353	}	668✔
354
355	if segmentKeysIndexMaxBytes > 0 {	3,688✔
356	if a, ok := seg.(*segment); ok {	3,688✔
357	a.buildIndex(segmentKeysIndexMaxBytes, segmentKeysIndexMinKeyBytes)	1,844✔
358	}	1,844✔
359	}
360
361	mref.SetExt(seg)	1,844✔
362	}
363
364	a[i] = mref.GetExt().(Segment)	7,314✔
365	}
366
367	f.ss = &segmentStack{	1,521✔
368	options: &options.CollectionOptions,	1,521✔
369	a: a,	1,521✔
370	refs: 1,	1,521✔
371	}	1,521✔
372		1,521✔
373	return mrefs, nil	1,521✔
374	}
375
376	// --------------------------------------------------------
377
378	// ChildCollectionNames returns an array of child collection name strings.
379	func (f *Footer) ChildCollectionNames() ([]string, error) {	×
380	var childNames = make([]string, len(f.ChildFooters))	×
381	idx := 0	×
382	for name := range f.ChildFooters {	×
383	childNames[idx] = name	×
384	idx++	×
385	}	×
386	return childNames, nil	×
387	}
388
389	// ChildCollectionSnapshot returns a Snapshot on a given child
390	// collection by its name.
391	func (f *Footer) ChildCollectionSnapshot(childCollectionName string) (
392	Snapshot, error) {	328✔
393	childFooter, exists := f.ChildFooters[childCollectionName]	328✔
394	if !exists {	398✔
395	return nil, nil	70✔
396	}	70✔
397	childFooter.AddRef()	258✔
398	return childFooter, nil	258✔
399	}
400
401	// Close decrements the ref count on this footer
402	func (f *Footer) Close() error {	166,036✔
403	f.DecRef()	166,036✔
404	return nil	166,036✔
405	}	166,036✔
406
407	// AddRef increases the ref count on this footer
408	func (f *Footer) AddRef() {	3,219✔
409	f.m.Lock()	3,219✔
410	f.refs++	3,219✔
411	f.m.Unlock()	3,219✔
412	}	3,219✔
413
414	// DecRef decreases the ref count on this footer
415	func (f *Footer) DecRef() {	4,369,894✔
416	f.m.Lock()	4,369,894✔
417	f.refs--	4,369,894✔
418	if f.refs <= 0 {	4,371,424✔
419	f.SegmentLocs.DecRef()	1,530✔
420	f.SegmentLocs = nil	1,530✔
421	f.ss = nil	1,530✔
422	}	1,530✔
423	f.m.Unlock()	4,373,850✔
424	}
425
426	// Length returns the length of this footer
427	func (f *Footer) Length() uint64 {	×
428	jBuf, err := json.Marshal(f)	×
429	if err != nil {	×
430	return 0	×
431	}	×
432
433	footerLen := footerBegLen + len(jBuf) + footerEndLen	×
434	return uint64(footerLen)	×
435	}
436
437	// --------------------------------------------------------
438
439	// segmentLocs returns the current SegmentLocs and segmentStack for
440	// a footer, while also incrementing the ref-count on the footer. The
441	// caller must DecRef() the footer when done.
442	func (f Footer) segmentLocs() (SegmentLocs, segmentStack) {	4,361,585✔
443	f.m.Lock()	4,361,585✔
444		4,361,585✔
445	f.refs++	4,361,585✔
446		4,361,585✔
447	slocs, ss := f.SegmentLocs, f.ss	4,361,585✔
448		4,361,585✔
449	f.m.Unlock()	4,361,585✔
450		4,361,585✔
451	return slocs, ss	4,361,585✔
452	}	4,361,585✔
453
454	// --------------------------------------------------------
455
456	// Get retrieves a val from the footer, and will return nil val
457	// if the entry does not exist in the footer.
458	func (f *Footer) Get(key []byte, readOptions ReadOptions) ([]byte, error) {	4,192,719✔
459	_, ss := f.segmentLocs()	4,192,719✔
460	if ss == nil {	4,192,719✔
461	f.DecRef()	×
462	return nil, nil	×
463	}	×
464
465	rv, err := ss.Get(key, readOptions)	4,197,804✔
466	if err == nil && rv != nil && !readOptions.NoCopyValue {	8,392,591✔
467	rv = append(make([]byte, 0, len(rv)), rv...) // Copy.	4,194,787✔
468	}	4,194,787✔
469
470	f.DecRef()	4,198,212✔
471		4,198,212✔
472	return rv, err	4,198,212✔
473	}
474
475	// StartIterator returns a new Iterator instance on this footer.
476	//
477	// On success, the returned Iterator will be positioned so that
478	// Iterator.Current() will either provide the first entry in the
479	// range or ErrIteratorDone.
480	//
481	// A startKeyIncl of nil means the logical "bottom-most" possible key
482	// and an endKeyExcl of nil means the logical "top-most" possible key.
483	func (f *Footer) StartIterator(startKeyIncl, endKeyExcl []byte,
484	iteratorOptions IteratorOptions) (Iterator, error) {	164,432✔
485	_, ss := f.segmentLocs()	164,432✔
486	if ss == nil {	164,432✔
487	f.DecRef()	×
488	return nil, nil	×
489	}	×
490
491	iter, err := ss.StartIterator(startKeyIncl, endKeyExcl, iteratorOptions)	164,432✔
492	if err != nil \|\| iter == nil {	164,432✔
493	f.DecRef()	×
494	return nil, err	×
495	}	×
496
497	initCloser, ok := iter.(InitCloser)	164,432✔
498	if !ok \|\| initCloser == nil {	164,432✔
499	iter.Close()	×
500	f.DecRef()	×
501	return nil, ErrUnexpected	×
502	}	×
503
504	err = initCloser.InitCloser(f)	164,432✔
505	if err != nil {	164,432✔
506	iter.Close()	×
507	f.DecRef()	×
508	return nil, err	×
509	}	×
510
511	return iter, nil	164,432✔
512	}

couchbase / moss / 306

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous