couchbase / moss / 299

pending completion

Build # 299

Build Type

push

travis-ci

Committed by

sreekanth-cb

Commit Message

<a href="https://github.com/couchbase/moss/commit/<a class=hub.com/couchbase/moss/commit/0947f8482ed9991a0480b29fee60685d9cc60cbf">0947f8482<a href="https://github.com/couchbase/moss/commit/0947f8482ed9991a0480b29fee60685d9cc60cbf">&quot;&gt;MB-27607 - mossStore should delete half-written files on an error

Logs to track files and files to remove on openStore

Change-Id: I</a><a class="double-link" href="https://github.com/couchbase/moss/commit/<a class="double-link" href="https://github.com/couchbase/moss/commit/5e3fd019f9553e4b7ea460413f191427d280761a">5e3fd019f</a>">5e3fd019f</a><a href="https://github.com/couchbase/moss/commit/0947f8482ed9991a0480b29fee60685d9cc60cbf">
Reviewed-on: http://review.couchbase.org/87967
Well-Formed: Build Bot &lt;build@couchbase.com&gt;
Reviewed-by: Sundar Sridharan &lt;sundar@couchbase.com&gt;
Reviewed-by: Steve Yen &lt;steve.yen@gmail.com&gt;
Tested-by: Sreekanth Sivasankaran <sreekanth.sivasankaran@couchbase.com>

Run Details

10 of 10 new or added lines in 1 file covered. (100.0%)

3348 of 4020 relevant lines covered (83.28%)

6550919.07 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

67.62

/store_footer.go

//  Copyright (c) 2016 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package moss

import (
        "bytes"
        "encoding/binary"
        "encoding/json"
        "fmt"
        "time"

        "github.com/edsrzf/mmap-go"
)

func (s *Store) persistFooter(file File, footer *Footer,
        options StorePersistOptions) error {
        startTime := time.Now()

        if !options.NoSync {
                err := file.Sync()
                if err != nil {
                        return err
                }
        }

        err := s.persistFooterUnsynced(file, footer)
        if err != nil {
                return err
        }

        if !options.NoSync {
                err = file.Sync()
        }

        if err == nil {
                s.histograms["PersistFooterUsecs"].Add(
                        uint64(time.Since(startTime).Nanoseconds()/1000), 1)
        }

        return err
}

func (s *Store) persistFooterUnsynced(file File, footer *Footer) error {
        jBuf, err := json.Marshal(footer)
        if err != nil {
                return err
        }

        finfo, err := file.Stat()
        if err != nil {
                return err
        }

        footerPos := pageAlignCeil(finfo.Size())
        footerLen := footerBegLen + len(jBuf) + footerEndLen

        footerBuf := bytes.NewBuffer(make([]byte, 0, footerLen))
        footerBuf.Write(StoreMagicBeg)
        footerBuf.Write(StoreMagicBeg)
        binary.Write(footerBuf, StoreEndian, uint32(StoreVersion))
        binary.Write(footerBuf, StoreEndian, uint32(footerLen))
        footerBuf.Write(jBuf)
        binary.Write(footerBuf, StoreEndian, footerPos)
        binary.Write(footerBuf, StoreEndian, uint32(footerLen))
        footerBuf.Write(StoreMagicEnd)
        footerBuf.Write(StoreMagicEnd)

        footerWritten, err := file.WriteAt(footerBuf.Bytes(), footerPos)
        if err != nil {
                return err
        }
        if footerWritten != len(footerBuf.Bytes()) {
                return fmt.Errorf("store: persistFooter error writing all footerBuf")
        }

        if AllocationGranularity != StorePageSize {
                // Some platforms (windows) only support mmap()'ing at an
                // allocation granularity that's != to a page size.
                //
                // However if on such platforms there are empty segments, then
                // due to the extra space imposed by the above granularity
                // requirement, mmap() can fail complaining about insufficient
                // file space.
                //
                // To avoid this error, simply pad up the file up to a page
                // boundary.  This pad of zeroes will not interfere with file
                // recovery.
                padding := make([]byte, int(pageAlignCeil(int64(footerWritten)))-footerWritten)
                _, err = file.WriteAt(padding, footerPos+int64(footerWritten))
                if err != nil {
                        return err
                }
        }

        footer.fileName = finfo.Name()
        footer.filePos = footerPos

        return nil
}

// --------------------------------------------------------

// ReadFooter reads the last valid Footer from a file.
func ReadFooter(options *StoreOptions, file File) (*Footer, error) {
        finfo, err := file.Stat()
        if err != nil {
                return nil, err
        }

        fref := &FileRef{file: file, refs: 1}

        // To avoid an EOF while reading, start scanning the footer from
        // the last byte. This is under the assumption that the footer is
        // at least 2 bytes long.
        f, err := ScanFooter(options, fref, finfo.Name(), finfo.Size()-1)
        if err != nil {
                return nil, err
        }

        fref.DecRef() // ScanFooter added its own ref-counts on success.

        return f, err
}

// --------------------------------------------------------

// ScanFooter scans a file backwards from the given pos for a valid
// Footer, adding ref-counts to fref on success.
func ScanFooter(options *StoreOptions, fref *FileRef, fileName string,
        pos int64) (*Footer, error) {
        footerBeg := make([]byte, footerBegLen)

        // Align pos to the start of a page (floor).
        pos = pageAlignFloor(pos)

        for {
                for { // Scan for StoreMagicBeg, which may be a potential footer.
                        if pos <= 0 {
                                return nil, ErrNoValidFooter
                        }

                        n, err := fref.file.ReadAt(footerBeg, pos)
                        if err != nil {
                                return nil, err
                        }

                        if n == footerBegLen &&
                                bytes.Equal(StoreMagicBeg, footerBeg[:lenMagicBeg]) &&
                                bytes.Equal(StoreMagicBeg, footerBeg[lenMagicBeg:2*lenMagicBeg]) {
                                break
                        }

                        // Move pos back by page size.
                        pos -= int64(StorePageSize)
                }

                // Read and check the potential footer.
                footerBegBuf := bytes.NewBuffer(footerBeg[2*lenMagicBeg:])

                var version uint32
                if err := binary.Read(footerBegBuf, StoreEndian, &version); err != nil {
                        return nil, err
                }
                if version != StoreVersion {
                        return nil, fmt.Errorf("store: version mismatch, "+
                                "current: %v != found: %v", StoreVersion, version)
                }

                var length uint32
                if err := binary.Read(footerBegBuf, StoreEndian, &length); err != nil {
                        return nil, err
                }

                data := make([]byte, int64(length)-int64(footerBegLen))

                n, err := fref.file.ReadAt(data, pos+int64(footerBegLen))
                if err != nil {
                        return nil, err
                }

                if n == len(data) &&
                        bytes.Equal(StoreMagicEnd, data[n-lenMagicEnd*2:n-lenMagicEnd]) &&
                        bytes.Equal(StoreMagicEnd, data[n-lenMagicEnd:]) {

                        content := int(length) - footerBegLen - footerEndLen
                        b := bytes.NewBuffer(data[content:])

                        var offset int64
                        if err = binary.Read(b, StoreEndian, &offset); err != nil {
                                return nil, err
                        }
                        if offset != pos {
                                return nil, fmt.Errorf("store: offset mismatch, "+
                                        "wanted: %v != found: %v", offset, pos)
                        }

                        var length1 uint32
                        if err = binary.Read(b, StoreEndian, &length1); err != nil {
                                return nil, err
                        }
                        if length1 != length {
                                return nil, fmt.Errorf("store: length mismatch, "+
                                        "wanted: %v != found: %v", length1, length)
                        }

                        f := &Footer{refs: 1, fileName: fileName, filePos: offset}

                        err = json.Unmarshal(data[:content], f)
                        if err != nil {
                                return nil, err
                        }

                        // json.Unmarshal would have just loaded the map.
                        // We now need to load each segment into the map.
                        // Also recursively load child footer segment stacks.
                        err = f.loadSegments(options, fref)
                        if err != nil {
                                return nil, err
                        }

                        return f, nil
                }
                // Else, invalid footer - StoreMagicEnd missing and/or file
                // pos out of bounds.

                // Footer was invalid, so keep scanning.
                pos -= int64(StorePageSize)
        }
}

// --------------------------------------------------------

// loadSegments() loads the segments of a footer.  Adds new ref-counts
// to the fref on success.  The footer will be in an already closed
// state on error.
func (f *Footer) loadSegments(options *StoreOptions, fref *FileRef) (err error) {
        // Track mrefs that we need to DecRef() if there's an error.
        mrefs := make([]*mmapRef, 0, len(f.SegmentLocs))
        mrefs, err = f.doLoadSegments(options, fref, mrefs)
        if err != nil {
                for _, mref := range mrefs {
                        mref.DecRef()
                }
                return err
        }

        return nil
}

func (f *Footer) doLoadSegments(options *StoreOptions, fref *FileRef,
        mrefs []*mmapRef) (mrefsSoFar []*mmapRef, err error) {
        // Recursively load the childFooters first.
        for _, childFooter := range f.ChildFooters {
                mrefs, err = childFooter.doLoadSegments(options, fref, mrefs)
                if err != nil {
                        return mrefs, err
                }
        }

        if f.ss != nil && f.ss.a != nil {
                return mrefs, nil
        }

        osFile := ToOsFile(fref.file)
        if osFile == nil {
                return mrefs, fmt.Errorf("store: doLoadSegments convert to os.File error")
        }

        a := make([]Segment, len(f.SegmentLocs))

        for i := range f.SegmentLocs {
                sloc := &f.SegmentLocs[i]

                mref := sloc.mref
                if mref != nil {
                        if mref.fref != fref {
                                return mrefs, fmt.Errorf("store: doLoadSegments fref mismatch")
                        }

                        mref.AddRef()
                        mrefs = append(mrefs, mref)
                } else {
                        // We persist kvs before buf, so KvsOffset < BufOffset.
                        begOffset := int64(sloc.KvsOffset)
                        endOffset := int64(sloc.BufOffset + sloc.BufBytes)

                        nbytes := int(endOffset - begOffset)

                        // Some platforms (windows) only support mmap()'ing at an
                        // allocation granularity that's != to a page size, so
                        // calculate the actual offset/nbytes to use.
                        begOffsetActual := pageOffset(begOffset, int64(AllocationGranularity))
                        begOffsetDelta := int(begOffset - begOffsetActual)
                        nbytesActual := nbytes + begOffsetDelta

                        mm, err := mmap.MapRegion(osFile, nbytesActual, mmap.RDONLY, 0, begOffsetActual)
                        if err != nil {
                                return mrefs,
                                        fmt.Errorf("store: doLoadSegments mmap.Map(), "+
                                                "begOffsetActual = %v,"+
                                                "nbytesActual = %v, sloc = %v, err: %v",
                                                begOffsetActual, nbytesActual, sloc, err)
                        }

                        fref.AddRef() // New mref owns 1 fref ref-count.

                        buf := mm[begOffsetDelta : begOffsetDelta+nbytes]

                        sloc.mref = &mmapRef{fref: fref, mm: mm, buf: buf, refs: 1}

                        mref = sloc.mref
                        mrefs = append(mrefs, mref)

                        segmentLoader, exists := SegmentLoaders[sloc.Kind]
                        if !exists || segmentLoader == nil {
                                return mrefs, fmt.Errorf("store: unknown SegmentLoc kind, sloc: %+v", sloc)
                        }

                        seg, err := segmentLoader(sloc)
                        if err != nil {
                                return mrefs, fmt.Errorf("store: segmentLoader failed, footer: %+v,"+
                                        " f.SegmentLocs: %+v, i: %d, options: %v err: %+v",
                                        f, f.SegmentLocs, i, options, err)
                        }

                        segmentKeysIndexMaxBytes := options.SegmentKeysIndexMaxBytes
                        if segmentKeysIndexMaxBytes == 0 {
                                segmentKeysIndexMaxBytes = DefaultStoreOptions.SegmentKeysIndexMaxBytes
                        }

                        segmentKeysIndexMinKeyBytes := options.SegmentKeysIndexMinKeyBytes
                        if segmentKeysIndexMinKeyBytes == 0 {
                                segmentKeysIndexMinKeyBytes = DefaultStoreOptions.SegmentKeysIndexMinKeyBytes
                        }

                        if segmentKeysIndexMaxBytes > 0 {
                                if a, ok := seg.(*segment); ok {
                                        a.buildIndex(segmentKeysIndexMaxBytes, segmentKeysIndexMinKeyBytes)
                                }
                        }

                        mref.SetExt(seg)
                }

                a[i] = mref.GetExt().(Segment)
        }

        f.ss = &segmentStack{
                options: &options.CollectionOptions,
                a:       a,
                refs:    1,
        }

        return mrefs, nil
}

// --------------------------------------------------------

// ChildCollectionNames returns an array of child collection name strings.
func (f *Footer) ChildCollectionNames() ([]string, error) {
        var childNames = make([]string, len(f.ChildFooters))
        idx := 0
        for name := range f.ChildFooters {
                childNames[idx] = name
                idx++
        }
        return childNames, nil
}

// ChildCollectionSnapshot returns a Snapshot on a given child
// collection by its name.
func (f *Footer) ChildCollectionSnapshot(childCollectionName string) (
        Snapshot, error) {
        childFooter, exists := f.ChildFooters[childCollectionName]
        if !exists {
                return nil, nil
        }
        childFooter.AddRef()
        return childFooter, nil
}

// Close decrements the ref count on this footer
func (f *Footer) Close() error {
        f.DecRef()
        return nil
}

// AddRef increases the ref count on this footer
func (f *Footer) AddRef() {
        f.m.Lock()
        f.refs++
        f.m.Unlock()
}

// DecRef decreases the ref count on this footer
func (f *Footer) DecRef() {
        f.m.Lock()
        f.refs--
        if f.refs <= 0 {
                f.SegmentLocs.DecRef()
                f.SegmentLocs = nil
                f.ss = nil
        }
        f.m.Unlock()
}

// Length returns the length of this footer
func (f *Footer) Length() uint64 {
        jBuf, err := json.Marshal(f)
        if err != nil {
                return 0
        }

        footerLen := footerBegLen + len(jBuf) + footerEndLen
        return uint64(footerLen)
}

// --------------------------------------------------------

// segmentLocs returns the current SegmentLocs and segmentStack for
// a footer, while also incrementing the ref-count on the footer.  The
// caller must DecRef() the footer when done.
func (f *Footer) segmentLocs() (SegmentLocs, *segmentStack) {
        f.m.Lock()

        f.refs++

        slocs, ss := f.SegmentLocs, f.ss

        f.m.Unlock()

        return slocs, ss
}

// --------------------------------------------------------

// Get retrieves a val from the footer, and will return nil val
// if the entry does not exist in the footer.
func (f *Footer) Get(key []byte, readOptions ReadOptions) ([]byte, error) {
        _, ss := f.segmentLocs()
        if ss == nil {
                f.DecRef()
                return nil, nil
        }

        rv, err := ss.Get(key, readOptions)
        if err == nil && rv != nil && !readOptions.NoCopyValue {
                rv = append(make([]byte, 0, len(rv)), rv...) // Copy.
        }

        f.DecRef()

        return rv, err
}

// StartIterator returns a new Iterator instance on this footer.
//
// On success, the returned Iterator will be positioned so that
// Iterator.Current() will either provide the first entry in the
// range or ErrIteratorDone.
//
// A startKeyIncl of nil means the logical "bottom-most" possible key
// and an endKeyExcl of nil means the logical "top-most" possible key.
func (f *Footer) StartIterator(startKeyIncl, endKeyExcl []byte,
        iteratorOptions IteratorOptions) (Iterator, error) {
        _, ss := f.segmentLocs()
        if ss == nil {
                f.DecRef()
                return nil, nil
        }

        iter, err := ss.StartIterator(startKeyIncl, endKeyExcl, iteratorOptions)
        if err != nil || iter == nil {
                f.DecRef()
                return nil, err
        }

        initCloser, ok := iter.(InitCloser)
        if !ok || initCloser == nil {
                iter.Close()
                f.DecRef()
                return nil, ErrUnexpected
        }

        err = initCloser.InitCloser(f)
        if err != nil {
                iter.Close()
                f.DecRef()
                return nil, err
        }

        return iter, nil
}

1	// Copyright (c) 2016 Couchbase, Inc.
2	// Licensed under the Apache License, Version 2.0 (the "License");
3	// you may not use this file except in compliance with the
4	// License. You may obtain a copy of the License at
5	// http://www.apache.org/licenses/LICENSE-2.0
6	// Unless required by applicable law or agreed to in writing,
7	// software distributed under the License is distributed on an "AS
8	// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
9	// express or implied. See the License for the specific language
10	// governing permissions and limitations under the License.
11
12	package moss
13
14	import (
15	"bytes"
16	"encoding/binary"
17	"encoding/json"
18	"fmt"
19	"time"
20
21	"github.com/edsrzf/mmap-go"
22	)
23
24	func (s Store) persistFooter(file File, footer Footer,
25	options StorePersistOptions) error {	1,158✔
26	startTime := time.Now()	1,158✔
27		1,158✔
28	if !options.NoSync {	2,315✔
29	err := file.Sync()	1,157✔
30	if err != nil {	1,157✔
31	return err	×
32	}	×
33	}
34
35	err := s.persistFooterUnsynced(file, footer)	1,158✔
36	if err != nil {	1,158✔
37	return err	×
38	}	×
39
40	if !options.NoSync {	2,315✔
41	err = file.Sync()	1,157✔
42	}	1,157✔
43
44	if err == nil {	2,316✔
45	s.histograms["PersistFooterUsecs"].Add(	1,158✔
46	uint64(time.Since(startTime).Nanoseconds()/1000), 1)	1,158✔
47	}	1,158✔
48
49	return err	1,158✔
50	}
51
52	func (s Store) persistFooterUnsynced(file File, footer Footer) error {	1,158✔
53	jBuf, err := json.Marshal(footer)	1,158✔
54	if err != nil {	1,158✔
55	return err	×
56	}	×
57
58	finfo, err := file.Stat()	1,158✔
59	if err != nil {	1,158✔
60	return err	×
61	}	×
62
63	footerPos := pageAlignCeil(finfo.Size())	1,158✔
64	footerLen := footerBegLen + len(jBuf) + footerEndLen	1,158✔
65		1,158✔
66	footerBuf := bytes.NewBuffer(make([]byte, 0, footerLen))	1,158✔
67	footerBuf.Write(StoreMagicBeg)	1,158✔
68	footerBuf.Write(StoreMagicBeg)	1,158✔
69	binary.Write(footerBuf, StoreEndian, uint32(StoreVersion))	1,158✔
70	binary.Write(footerBuf, StoreEndian, uint32(footerLen))	1,158✔
71	footerBuf.Write(jBuf)	1,158✔
72	binary.Write(footerBuf, StoreEndian, footerPos)	1,158✔
73	binary.Write(footerBuf, StoreEndian, uint32(footerLen))	1,158✔
74	footerBuf.Write(StoreMagicEnd)	1,158✔
75	footerBuf.Write(StoreMagicEnd)	1,158✔
76		1,158✔
77	footerWritten, err := file.WriteAt(footerBuf.Bytes(), footerPos)	1,158✔
78	if err != nil {	1,158✔
79	return err	×
80	}	×
81	if footerWritten != len(footerBuf.Bytes()) {	1,158✔
82	return fmt.Errorf("store: persistFooter error writing all footerBuf")	×
83	}	×
84
85	if AllocationGranularity != StorePageSize {	1,158✔
86	// Some platforms (windows) only support mmap()'ing at an	×
87	// allocation granularity that's != to a page size.	×
88	//	×
89	// However if on such platforms there are empty segments, then	×
90	// due to the extra space imposed by the above granularity	×
91	// requirement, mmap() can fail complaining about insufficient	×
92	// file space.	×
93	//	×
94	// To avoid this error, simply pad up the file up to a page	×
95	// boundary. This pad of zeroes will not interfere with file	×
96	// recovery.	×
97	padding := make([]byte, int(pageAlignCeil(int64(footerWritten)))-footerWritten)	×
98	_, err = file.WriteAt(padding, footerPos+int64(footerWritten))	×
99	if err != nil {	×
100	return err	×
101	}	×
102	}
103
104	footer.fileName = finfo.Name()	1,158✔
105	footer.filePos = footerPos	1,158✔
106		1,158✔
107	return nil	1,158✔
108	}
109
110	// --------------------------------------------------------
111
112	// ReadFooter reads the last valid Footer from a file.
113	func ReadFooter(options StoreOptions, file File) (Footer, error) {	21✔
114	finfo, err := file.Stat()	21✔
115	if err != nil {	21✔
116	return nil, err	×
117	}	×
118
119	fref := &FileRef{file: file, refs: 1}	21✔
120		21✔
121	// To avoid an EOF while reading, start scanning the footer from	21✔
122	// the last byte. This is under the assumption that the footer is	21✔
123	// at least 2 bytes long.	21✔
124	f, err := ScanFooter(options, fref, finfo.Name(), finfo.Size()-1)	21✔
125	if err != nil {	25✔
126	return nil, err	4✔
127	}	4✔
128
129	fref.DecRef() // ScanFooter added its own ref-counts on success.	17✔
130		17✔
131	return f, err	17✔
132	}
133
134	// --------------------------------------------------------
135
136	// ScanFooter scans a file backwards from the given pos for a valid
137	// Footer, adding ref-counts to fref on success.
138	func ScanFooter(options StoreOptions, fref FileRef, fileName string,
139	pos int64) (*Footer, error) {	24✔
140	footerBeg := make([]byte, footerBegLen)	24✔
141		24✔
142	// Align pos to the start of a page (floor).	24✔
143	pos = pageAlignFloor(pos)	24✔
144		24✔
145	for {	48✔
146	for { // Scan for StoreMagicBeg, which may be a potential footer.	48✔
147	if pos <= 0 {	29✔
148	return nil, ErrNoValidFooter	5✔
149	}	5✔
150
151	n, err := fref.file.ReadAt(footerBeg, pos)	19✔
152	if err != nil {	19✔
153	return nil, err	×
154	}	×
155
156	if n == footerBegLen &&	19✔
157	bytes.Equal(StoreMagicBeg, footerBeg[:lenMagicBeg]) &&	19✔
158	bytes.Equal(StoreMagicBeg, footerBeg[lenMagicBeg:2*lenMagicBeg]) {	38✔
159	break	19✔
160	}
161
162	// Move pos back by page size.
163	pos -= int64(StorePageSize)	×
164	}
165
166	// Read and check the potential footer.
167	footerBegBuf := bytes.NewBuffer(footerBeg[2*lenMagicBeg:])	19✔
168		19✔
169	var version uint32	19✔
170	if err := binary.Read(footerBegBuf, StoreEndian, &version); err != nil {	19✔
171	return nil, err	×
172	}	×
173	if version != StoreVersion {	19✔
174	return nil, fmt.Errorf("store: version mismatch, "+	×
175	"current: %v != found: %v", StoreVersion, version)	×
176	}	×
177
178	var length uint32	19✔
179	if err := binary.Read(footerBegBuf, StoreEndian, &length); err != nil {	19✔
180	return nil, err	×
181	}	×
182
183	data := make([]byte, int64(length)-int64(footerBegLen))	19✔
184		19✔
185	n, err := fref.file.ReadAt(data, pos+int64(footerBegLen))	19✔
186	if err != nil {	19✔
187	return nil, err	×
188	}	×
189
190	if n == len(data) &&	19✔
191	bytes.Equal(StoreMagicEnd, data[n-lenMagicEnd*2:n-lenMagicEnd]) &&	19✔
192	bytes.Equal(StoreMagicEnd, data[n-lenMagicEnd:]) {	38✔
193		19✔
194	content := int(length) - footerBegLen - footerEndLen	19✔
195	b := bytes.NewBuffer(data[content:])	19✔
196		19✔
197	var offset int64	19✔
198	if err = binary.Read(b, StoreEndian, &offset); err != nil {	19✔
199	return nil, err	×
200	}	×
201	if offset != pos {	19✔
202	return nil, fmt.Errorf("store: offset mismatch, "+	×
203	"wanted: %v != found: %v", offset, pos)	×
204	}	×
205
206	var length1 uint32	19✔
207	if err = binary.Read(b, StoreEndian, &length1); err != nil {	19✔
208	return nil, err	×
209	}	×
210	if length1 != length {	19✔
211	return nil, fmt.Errorf("store: length mismatch, "+	×
212	"wanted: %v != found: %v", length1, length)	×
213	}	×
214
215	f := &Footer{refs: 1, fileName: fileName, filePos: offset}	19✔
216		19✔
217	err = json.Unmarshal(data[:content], f)	19✔
218	if err != nil {	19✔
219	return nil, err	×
220	}	×
221
222	// json.Unmarshal would have just loaded the map.
223	// We now need to load each segment into the map.
224	// Also recursively load child footer segment stacks.
225	err = f.loadSegments(options, fref)	19✔
226	if err != nil {	19✔
227	return nil, err	×
228	}	×
229
230	return f, nil	19✔
231	}
232	// Else, invalid footer - StoreMagicEnd missing and/or file
233	// pos out of bounds.
234
235	// Footer was invalid, so keep scanning.
236	pos -= int64(StorePageSize)	×
237	}
238	}
239
240	// --------------------------------------------------------
241
242	// loadSegments() loads the segments of a footer. Adds new ref-counts
243	// to the fref on success. The footer will be in an already closed
244	// state on error.
245	func (f Footer) loadSegments(options StoreOptions, fref *FileRef) (err error) {	1,176✔
246	// Track mrefs that we need to DecRef() if there's an error.	1,176✔
247	mrefs := make([]*mmapRef, 0, len(f.SegmentLocs))	1,176✔
248	mrefs, err = f.doLoadSegments(options, fref, mrefs)	1,176✔
249	if err != nil {	1,176✔
250	for _, mref := range mrefs {	×
251	mref.DecRef()	×
252	}	×
253	return err	×
254	}
255
256	return nil	1,176✔
257	}
258
259	func (f Footer) doLoadSegments(options StoreOptions, fref *FileRef,
260	mrefs []mmapRef) (mrefsSoFar []mmapRef, err error) {	1,224✔
261	// Recursively load the childFooters first.	1,224✔
262	for _, childFooter := range f.ChildFooters {	1,272✔
263	mrefs, err = childFooter.doLoadSegments(options, fref, mrefs)	48✔
264	if err != nil {	48✔
265	return mrefs, err	×
266	}	×
267	}
268
269	if f.ss != nil && f.ss.a != nil {	1,224✔
270	return mrefs, nil	×
271	}	×
272
273	osFile := ToOsFile(fref.file)	1,224✔
274	if osFile == nil {	1,224✔
275	return mrefs, fmt.Errorf("store: doLoadSegments convert to os.File error")	×
276	}	×
277
278	a := make([]Segment, len(f.SegmentLocs))	1,224✔
279		1,224✔
280	for i := range f.SegmentLocs {	6,846✔
281	sloc := &f.SegmentLocs[i]	5,622✔
282		5,622✔
283	mref := sloc.mref	5,622✔
284	if mref != nil {	9,803✔
285	if mref.fref != fref {	4,181✔
286	return mrefs, fmt.Errorf("store: doLoadSegments fref mismatch")	×
287	}	×
288
289	mref.AddRef()	4,181✔
290	mrefs = append(mrefs, mref)	4,181✔
291	} else {	1,441✔
292	// We persist kvs before buf, so KvsOffset < BufOffset.	1,441✔
293	begOffset := int64(sloc.KvsOffset)	1,441✔
294	endOffset := int64(sloc.BufOffset + sloc.BufBytes)	1,441✔
295		1,441✔
296	nbytes := int(endOffset - begOffset)	1,441✔
297		1,441✔
298	// Some platforms (windows) only support mmap()'ing at an	1,441✔
299	// allocation granularity that's != to a page size, so	1,441✔
300	// calculate the actual offset/nbytes to use.	1,441✔
301	begOffsetActual := pageOffset(begOffset, int64(AllocationGranularity))	1,441✔
302	begOffsetDelta := int(begOffset - begOffsetActual)	1,441✔
303	nbytesActual := nbytes + begOffsetDelta	1,441✔
304		1,441✔
305	mm, err := mmap.MapRegion(osFile, nbytesActual, mmap.RDONLY, 0, begOffsetActual)	1,441✔
306	if err != nil {	1,441✔
307	return mrefs,	×
308	fmt.Errorf("store: doLoadSegments mmap.Map(), "+	×
309	"begOffsetActual = %v,"+	×
310	"nbytesActual = %v, sloc = %v, err: %v",	×
311	begOffsetActual, nbytesActual, sloc, err)	×
312	}	×
313
314	fref.AddRef() // New mref owns 1 fref ref-count.	1,441✔
315		1,441✔
316	buf := mm[begOffsetDelta : begOffsetDelta+nbytes]	1,441✔
317		1,441✔
318	sloc.mref = &mmapRef{fref: fref, mm: mm, buf: buf, refs: 1}	1,441✔
319		1,441✔
320	mref = sloc.mref	1,441✔
321	mrefs = append(mrefs, mref)	1,441✔
322		1,441✔
323	segmentLoader, exists := SegmentLoaders[sloc.Kind]	1,441✔
324	if !exists \|\| segmentLoader == nil {	1,441✔
325	return mrefs, fmt.Errorf("store: unknown SegmentLoc kind, sloc: %+v", sloc)	×
326	}	×
327
328	seg, err := segmentLoader(sloc)	1,441✔
329	if err != nil {	1,441✔
330	return mrefs, fmt.Errorf("store: segmentLoader failed, footer: %+v,"+	×
331	" f.SegmentLocs: %+v, i: %d, options: %v err: %+v",	×
332	f, f.SegmentLocs, i, options, err)	×
333	}	×
334
335	segmentKeysIndexMaxBytes := options.SegmentKeysIndexMaxBytes	1,441✔
336	if segmentKeysIndexMaxBytes == 0 {	1,933✔
337	segmentKeysIndexMaxBytes = DefaultStoreOptions.SegmentKeysIndexMaxBytes	492✔
338	}	492✔
339
340	segmentKeysIndexMinKeyBytes := options.SegmentKeysIndexMinKeyBytes	1,441✔
341	if segmentKeysIndexMinKeyBytes == 0 {	1,933✔
342	segmentKeysIndexMinKeyBytes = DefaultStoreOptions.SegmentKeysIndexMinKeyBytes	492✔
343	}	492✔
344
345	if segmentKeysIndexMaxBytes > 0 {	2,882✔
346	if a, ok := seg.(*segment); ok {	2,882✔
347	a.buildIndex(segmentKeysIndexMaxBytes, segmentKeysIndexMinKeyBytes)	1,441✔
348	}	1,441✔
349	}
350
351	mref.SetExt(seg)	1,441✔
352	}
353
354	a[i] = mref.GetExt().(Segment)	5,622✔
355	}
356
357	f.ss = &segmentStack{	1,224✔
358	options: &options.CollectionOptions,	1,224✔
359	a: a,	1,224✔
360	refs: 1,	1,224✔
361	}	1,224✔
362		1,224✔
363	return mrefs, nil	1,224✔
364	}
365
366	// --------------------------------------------------------
367
368	// ChildCollectionNames returns an array of child collection name strings.
369	func (f *Footer) ChildCollectionNames() ([]string, error) {	×
370	var childNames = make([]string, len(f.ChildFooters))	×
371	idx := 0	×
372	for name := range f.ChildFooters {	×
373	childNames[idx] = name	×
374	idx++	×
375	}	×
376	return childNames, nil	×
377	}
378
379	// ChildCollectionSnapshot returns a Snapshot on a given child
380	// collection by its name.
381	func (f *Footer) ChildCollectionSnapshot(childCollectionName string) (
382	Snapshot, error) {	337✔
383	childFooter, exists := f.ChildFooters[childCollectionName]	337✔
384	if !exists {	392✔
385	return nil, nil	55✔
386	}	55✔
387	childFooter.AddRef()	282✔
388	return childFooter, nil	282✔
389	}
390
391	// Close decrements the ref count on this footer
392	func (f *Footer) Close() error {	143,024✔
393	f.DecRef()	143,024✔
394	return nil	143,024✔
395	}	143,024✔
396
397	// AddRef increases the ref count on this footer
398	func (f *Footer) AddRef() {	2,637✔
399	f.m.Lock()	2,637✔
400	f.refs++	2,637✔
401	f.m.Unlock()	2,637✔
402	}	2,637✔
403
404	// DecRef decreases the ref count on this footer
405	func (f *Footer) DecRef() {	4,144,205✔
406	f.m.Lock()	4,144,205✔
407	f.refs--	4,144,205✔
408	if f.refs <= 0 {	4,145,421✔
409	f.SegmentLocs.DecRef()	1,216✔
410	f.SegmentLocs = nil	1,216✔
411	f.ss = nil	1,216✔
412	}	1,216✔
413	f.m.Unlock()	4,144,547✔
414	}
415
416	// Length returns the length of this footer
417	func (f *Footer) Length() uint64 {	×
418	jBuf, err := json.Marshal(f)	×
419	if err != nil {	×
420	return 0	×
421	}	×
422
423	footerLen := footerBegLen + len(jBuf) + footerEndLen	×
424	return uint64(footerLen)	×
425	}
426
427	// --------------------------------------------------------
428
429	// segmentLocs returns the current SegmentLocs and segmentStack for
430	// a footer, while also incrementing the ref-count on the footer. The
431	// caller must DecRef() the footer when done.
432	func (f Footer) segmentLocs() (SegmentLocs, segmentStack) {	4,140,195✔
433	f.m.Lock()	4,140,195✔
434		4,140,195✔
435	f.refs++	4,140,195✔
436		4,140,195✔
437	slocs, ss := f.SegmentLocs, f.ss	4,140,195✔
438		4,140,195✔
439	f.m.Unlock()	4,140,195✔
440		4,140,195✔
441	return slocs, ss	4,140,195✔
442	}	4,140,195✔
443
444	// --------------------------------------------------------
445
446	// Get retrieves a val from the footer, and will return nil val
447	// if the entry does not exist in the footer.
448	func (f *Footer) Get(key []byte, readOptions ReadOptions) ([]byte, error) {	3,995,784✔
449	_, ss := f.segmentLocs()	3,995,784✔
450	if ss == nil {	3,995,784✔
451	f.DecRef()	×
452	return nil, nil	×
453	}	×
454
455	rv, err := ss.Get(key, readOptions)	3,998,571✔
456	if err == nil && rv != nil && !readOptions.NoCopyValue {	7,990,597✔
457	rv = append(make([]byte, 0, len(rv)), rv...) // Copy.	3,992,026✔
458	}	3,992,026✔
459
460	f.DecRef()	3,996,447✔
461		3,996,447✔
462	return rv, err	3,996,447✔
463	}
464
465	// StartIterator returns a new Iterator instance on this footer.
466	//
467	// On success, the returned Iterator will be positioned so that
468	// Iterator.Current() will either provide the first entry in the
469	// range or ErrIteratorDone.
470	//
471	// A startKeyIncl of nil means the logical "bottom-most" possible key
472	// and an endKeyExcl of nil means the logical "top-most" possible key.
473	func (f *Footer) StartIterator(startKeyIncl, endKeyExcl []byte,
474	iteratorOptions IteratorOptions) (Iterator, error) {	141,735✔
475	_, ss := f.segmentLocs()	141,735✔
476	if ss == nil {	141,735✔
477	f.DecRef()	×
478	return nil, nil	×
479	}	×
480
481	iter, err := ss.StartIterator(startKeyIncl, endKeyExcl, iteratorOptions)	141,735✔
482	if err != nil \|\| iter == nil {	141,735✔
483	f.DecRef()	×
484	return nil, err	×
485	}	×
486
487	initCloser, ok := iter.(InitCloser)	141,735✔
488	if !ok \|\| initCloser == nil {	141,735✔
489	iter.Close()	×
490	f.DecRef()	×
491	return nil, ErrUnexpected	×
492	}	×
493
494	err = initCloser.InitCloser(f)	141,735✔
495	if err != nil {	141,735✔
496	iter.Close()	×
497	f.DecRef()	×
498	return nil, err	×
499	}	×
500
501	return iter, nil	141,735✔
502	}

couchbase / moss / 299

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous