6539229776

Committed 16 Oct 2023 09:05PM UTC coverage: 71.743% (-2.4%) from 74.097%

Build # 6539229776

Build Type

Pull #357

github

Committed by

jstaf

Commit Message

0.14.0-2 dummy release to force gpg key refresh on OBS

Pull Request Pull Request #357: 0.14.0-2 dummy release to force gpg key refresh on OBS

Run Details

1828 of 2548 relevant lines covered (71.74%)

172113.74 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

70.17

/fs/cache.go

package fs

import (
        "errors"
        "fmt"
        "os"
        "path/filepath"
        "strings"
        "sync"
        "time"

        "github.com/hanwen/go-fuse/v2/fuse"
        "github.com/jstaf/onedriver/fs/graph"
        "github.com/rs/zerolog/log"
        bolt "go.etcd.io/bbolt"
)

// Filesystem is the actual FUSE filesystem and uses the go analogy of the
// "low-level" FUSE API here:
// https://github.com/libfuse/libfuse/blob/master/include/fuse_lowlevel.h
type Filesystem struct {
        fuse.RawFileSystem

        metadata  sync.Map
        db        *bolt.DB
        content   *LoopbackCache
        auth      *graph.Auth
        root      string // the id of the filesystem's root item
        deltaLink string
        uploads   *UploadManager

        sync.RWMutex
        offline    bool
        lastNodeID uint64
        inodes     []string

        // tracks currently open directories
        opendirsM sync.RWMutex
        opendirs  map[uint64][]*Inode
}

// boltdb buckets
var (
        bucketContent  = []byte("content")
        bucketMetadata = []byte("metadata")
        bucketDelta    = []byte("delta")
        bucketVersion  = []byte("version")
)

// so we can tell what format the db has
const fsVersion = "1"

// NewFilesystem creates a new filesystem
func NewFilesystem(auth *graph.Auth, cacheDir string) *Filesystem {
        // prepare cache directory
        if _, err := os.Stat(cacheDir); err != nil {
                if err = os.Mkdir(cacheDir, 0700); err != nil {
                        log.Fatal().Err(err).Msg("Could not create cache directory.")
                }
        }
        db, err := bolt.Open(
                filepath.Join(cacheDir, "onedriver.db"),
                0600,
                &bolt.Options{Timeout: time.Second * 5},
        )
        if err != nil {
                log.Fatal().Err(err).Msg("Could not open DB. Is it already in use by another mount?")
        }

        content := NewLoopbackCache(filepath.Join(cacheDir, "content"))
        db.Update(func(tx *bolt.Tx) error {
                tx.CreateBucketIfNotExists(bucketMetadata)
                tx.CreateBucketIfNotExists(bucketDelta)
                versionBucket, _ := tx.CreateBucketIfNotExists(bucketVersion)

                // migrate old content bucket to the local filesystem
                b := tx.Bucket(bucketContent)
                if b != nil {
                        oldVersion := "0"
                        log.Info().
                                Str("oldVersion", oldVersion).
                                Str("version", fsVersion).
                                Msg("Migrating to new db format.")
                        err := b.ForEach(func(k []byte, v []byte) error {
                                log.Info().Bytes("key", k).Msg("Migrating file content.")
                                if err := content.Insert(string(k), v); err != nil {
                                        return err
                                }
                                return b.Delete(k)
                        })
                        if err != nil {
                                log.Error().Err(err).Msg("Migration failed.")
                        }
                        tx.DeleteBucket(bucketContent)
                        log.Info().
                                Str("oldVersion", oldVersion).
                                Str("version", fsVersion).
                                Msg("Migrations complete.")
                }
                return versionBucket.Put([]byte("version"), []byte(fsVersion))
        })

        // ok, ready to start fs
        fs := &Filesystem{
                RawFileSystem: fuse.NewDefaultRawFileSystem(),
                content:       content,
                db:            db,
                auth:          auth,
                opendirs:      make(map[uint64][]*Inode),
        }

        rootItem, err := graph.GetItem("root", auth)
        root := NewInodeDriveItem(rootItem)
        if err != nil {
                if graph.IsOffline(err) {
                        // no network, load from db if possible and go to read-only state
                        fs.Lock()
                        fs.offline = true
                        fs.Unlock()
                        if root = fs.GetID("root"); root == nil {
                                log.Fatal().Msg(
                                        "We are offline and could not fetch the filesystem root item from disk.",
                                )
                        }
                        // when offline, we load the cache deltaLink from disk
                        fs.db.View(func(tx *bolt.Tx) error {
                                if link := tx.Bucket(bucketDelta).Get([]byte("deltaLink")); link != nil {
                                        fs.deltaLink = string(link)
                                } else {
                                        // Only reached if a previous online session never survived
                                        // long enough to save its delta link. We explicitly disallow these
                                        // types of startups as it's possible for things to get out of sync
                                        // this way.
                                        log.Fatal().Msg("Cannot perform an offline startup without a valid " +
                                                "delta link from a previous session.")
                                }
                                return nil
                        })
                } else {
                        log.Fatal().Err(err).Msg("Could not fetch root item of filesystem!")
                }
        }
        // root inode is inode 1
        fs.root = root.ID()
        fs.InsertID(fs.root, root)

        fs.uploads = NewUploadManager(2*time.Second, db, fs, auth)

        if !fs.IsOffline() {
                // .Trash-UID is used by "gio trash" for user trash, create it if it
                // does not exist
                trash := fmt.Sprintf(".Trash-%d", os.Getuid())
                if child, _ := fs.GetChild(fs.root, trash, auth); child == nil {
                        item, err := graph.Mkdir(trash, fs.root, auth)
                        if err != nil {
                                log.Error().Err(err).
                                        Msg("Could not create trash folder. " +
                                                "Trashing items through the file browser may result in errors.")
                        } else {
                                fs.InsertID(item.ID, NewInodeDriveItem(item))
                        }
                }

                // using token=latest because we don't care about existing items - they'll
                // be downloaded on-demand by the cache
                fs.deltaLink = "/me/drive/root/delta?token=latest"
        }

        // deltaloop is started manually
        return fs
}

// IsOffline returns whether or not the cache thinks its offline.
func (f *Filesystem) IsOffline() bool {
        f.RLock()
        defer f.RUnlock()
        return f.offline
}

// TranslateID returns the DriveItemID for a given NodeID
func (f *Filesystem) TranslateID(nodeID uint64) string {
        f.RLock()
        defer f.RUnlock()
        if nodeID > f.lastNodeID || nodeID == 0 {
                return ""
        }
        return f.inodes[nodeID-1]
}

// GetNodeID fetches the inode for a particular inode ID.
func (f *Filesystem) GetNodeID(nodeID uint64) *Inode {
        id := f.TranslateID(nodeID)
        if id == "" {
                return nil
        }
        return f.GetID(id)
}

// InsertNodeID assigns a numeric inode ID used by the kernel if one is not
// already assigned.
func (f *Filesystem) InsertNodeID(inode *Inode) uint64 {
        nodeID := inode.NodeID()
        if nodeID == 0 {
                // lock ordering is to satisfy deadlock detector
                inode.Lock()
                f.Lock()

                f.lastNodeID++
                f.inodes = append(f.inodes, inode.DriveItem.ID)
                nodeID = f.lastNodeID
                inode.nodeID = nodeID

                f.Unlock()
                inode.Unlock()
        }
        return nodeID
}

// GetID gets an inode from the cache by ID. No API fetching is performed.
// Result is nil if no inode is found.
func (f *Filesystem) GetID(id string) *Inode {
        entry, exists := f.metadata.Load(id)
        if !exists {
                // we allow fetching from disk as a fallback while offline (and it's also
                // necessary while transitioning from offline->online)
                var found *Inode
                f.db.View(func(tx *bolt.Tx) error {
                        data := tx.Bucket(bucketMetadata).Get([]byte(id))
                        var err error
                        if data != nil {
                                found, err = NewInodeJSON(data)
                        }
                        return err
                })
                if found != nil {
                        f.InsertNodeID(found)
                        f.metadata.Store(id, found) // move to memory for next time
                }
                return found
        }
        return entry.(*Inode)
}

// InsertID inserts a single item into the filesystem by ID and sets its parent
// using the Inode.Parent.ID, if set. Must be called after DeleteID, if being
// used to rename/move an item. This is the main way new Inodes are added to the
// filesystem. Returns the Inode's numeric NodeID.
func (f *Filesystem) InsertID(id string, inode *Inode) uint64 {
        f.metadata.Store(id, inode)
        nodeID := f.InsertNodeID(inode)

        if id != inode.ID() {
                // we update the inode IDs here in case they do not match/changed
                inode.Lock()
                inode.DriveItem.ID = id
                inode.Unlock()

                f.Lock()
                if nodeID <= f.lastNodeID {
                        f.inodes[nodeID-1] = id
                } else {
                        log.Error().
                                Uint64("nodeID", nodeID).
                                Uint64("lastNodeID", f.lastNodeID).
                                Msg("NodeID exceeded maximum node ID! Ignoring ID change.")
                }
                f.Unlock()
        }

        parentID := inode.ParentID()
        if parentID == "" {
                // root item, or parent not set
                return nodeID
        }
        parent := f.GetID(parentID)
        if parent == nil {
                log.Error().
                        Str("parentID", parentID).
                        Str("childID", id).
                        Str("childName", inode.Name()).
                        Msg("Parent item could not be found when setting parent.")
                return nodeID
        }

        // check if the item has already been added to the parent
        // Lock order is super key here, must go parent->child or the deadlock
        // detector screams at us.
        parent.Lock()
        defer parent.Unlock()
        for _, child := range parent.children {
                if child == id {
                        // exit early, child cannot be added twice
                        return nodeID
                }
        }

        // add to parent
        if inode.IsDir() {
                parent.subdir++
        }
        parent.children = append(parent.children, id)

        return nodeID
}

// InsertChild adds an item as a child of a specified parent ID.
func (f *Filesystem) InsertChild(parentID string, child *Inode) uint64 {
        child.Lock()
        // should already be set, just double-checking here.
        child.DriveItem.Parent.ID = parentID
        id := child.DriveItem.ID
        child.Unlock()
        return f.InsertID(id, child)
}

// DeleteID deletes an item from the cache, and removes it from its parent. Must
// be called before InsertID if being used to rename/move an item.
func (f *Filesystem) DeleteID(id string) {
        if inode := f.GetID(id); inode != nil {
                parent := f.GetID(inode.ParentID())
                parent.Lock()
                for i, childID := range parent.children {
                        if childID == id {
                                parent.children = append(parent.children[:i], parent.children[i+1:]...)
                                if inode.IsDir() {
                                        parent.subdir--
                                }
                                break
                        }
                }
                parent.Unlock()
        }
        f.metadata.Delete(id)
        f.uploads.CancelUpload(id)
}

// GetChild fetches a named child of an item. Wraps GetChildrenID.
func (f *Filesystem) GetChild(id string, name string, auth *graph.Auth) (*Inode, error) {
        children, err := f.GetChildrenID(id, auth)
        if err != nil {
                return nil, err
        }
        for _, child := range children {
                if strings.EqualFold(child.Name(), name) {
                        return child, nil
                }
        }
        return nil, errors.New("child does not exist")
}

// GetChildrenID grabs all DriveItems that are the children of the given ID. If
// items are not found, they are fetched.
func (f *Filesystem) GetChildrenID(id string, auth *graph.Auth) (map[string]*Inode, error) {
        // fetch item and catch common errors
        inode := f.GetID(id)
        children := make(map[string]*Inode)
        if inode == nil {
                log.Error().Str("id", id).Msg("Inode not found in cache")
                return children, errors.New(id + " not found in cache")
        } else if !inode.IsDir() {
                // Normal files are treated as empty folders. This only gets called if
                // we messed up and tried to get the children of a plain-old file.
                log.Warn().
                        Str("id", id).
                        Str("path", inode.Path()).
                        Msg("Attepted to get children of ordinary file")
                return children, nil
        }

        // If item.children is not nil, it means we have the item's children
        // already and can fetch them directly from the cache
        inode.RLock()
        if inode.children != nil {
                // can potentially have out-of-date child metadata if started offline, but since
                // changes are disallowed while offline, the children will be back in sync after
                // the first successful delta fetch (which also brings the fs back online)
                for _, childID := range inode.children {
                        child := f.GetID(childID)
                        if child == nil {
                                // will be nil if deleted or never existed
                                continue
                        }
                        children[strings.ToLower(child.Name())] = child
                }
                inode.RUnlock()
                return children, nil
        }
        inode.RUnlock()

        // We haven't fetched the children for this item yet, get them from the server.
        fetched, err := graph.GetItemChildren(id, auth)
        if err != nil {
                if graph.IsOffline(err) {
                        log.Warn().Str("id", id).
                                Msg("We are offline, and no children found in cache. " +
                                        "Pretending there are no children.")
                        return children, nil
                }
                // something else happened besides being offline
                return nil, err
        }

        inode.Lock()
        inode.children = make([]string, 0)
        for _, item := range fetched {
                // we will always have an id after fetching from the server
                child := NewInodeDriveItem(item)
                f.InsertNodeID(child)
                f.metadata.Store(child.DriveItem.ID, child)

                // store in result map
                children[strings.ToLower(child.Name())] = child

                // store id in parent item and increment parents subdirectory count
                inode.children = append(inode.children, child.DriveItem.ID)
                if child.IsDir() {
                        inode.subdir++
                }
        }
        inode.Unlock()

        return children, nil
}

// GetChildrenPath grabs all DriveItems that are the children of the resource at
// the path. If items are not found, they are fetched.
func (f *Filesystem) GetChildrenPath(path string, auth *graph.Auth) (map[string]*Inode, error) {
        inode, err := f.GetPath(path, auth)
        if err != nil {
                return make(map[string]*Inode), err
        }
        return f.GetChildrenID(inode.ID(), auth)
}

// GetPath fetches a given DriveItem in the cache, if any items along the way are
// not found, they are fetched.
func (f *Filesystem) GetPath(path string, auth *graph.Auth) (*Inode, error) {
        lastID := f.root
        if path == "/" {
                return f.GetID(lastID), nil
        }

        // from the root directory, traverse the chain of items till we reach our
        // target ID.
        path = strings.TrimSuffix(strings.ToLower(path), "/")
        split := strings.Split(path, "/")[1:] //omit leading "/"
        var inode *Inode
        for i := 0; i < len(split); i++ {
                // fetches children
                children, err := f.GetChildrenID(lastID, auth)
                if err != nil {
                        return nil, err
                }

                var exists bool // if we use ":=", item is shadowed
                inode, exists = children[split[i]]
                if !exists {
                        // the item still doesn't exist after fetching from server. it
                        // doesn't exist
                        return nil, errors.New(strings.Join(split[:i+1], "/") +
                                " does not exist on server or in local cache")
                }
                lastID = inode.ID()
        }
        return inode, nil
}

// DeletePath an item from the cache by path. Must be called before Insert if
// being used to move/rename an item.
func (f *Filesystem) DeletePath(key string) {
        inode, _ := f.GetPath(strings.ToLower(key), nil)
        if inode != nil {
                f.DeleteID(inode.ID())
        }
}

// InsertPath lets us manually insert an item to the cache (like if it was
// created locally). Overwrites a cached item if present. Must be called after
// delete if being used to move/rename an item.
func (f *Filesystem) InsertPath(key string, auth *graph.Auth, inode *Inode) (uint64, error) {
        key = strings.ToLower(key)

        // set the item.Parent.ID properly if the item hasn't been in the cache
        // before or is being moved.
        parent, err := f.GetPath(filepath.Dir(key), auth)
        if err != nil {
                return 0, err
        } else if parent == nil {
                const errMsg string = "parent of key was nil"
                log.Error().
                        Str("key", key).
                        Str("path", inode.Path()).
                        Msg(errMsg)
                return 0, errors.New(errMsg)
        }

        // Coded this way to make sure locks are in the same order for the deadlock
        // detector (lock ordering needs to be the same as InsertID: Parent->Child).
        parentID := parent.ID()
        inode.Lock()
        inode.DriveItem.Parent.ID = parentID
        id := inode.DriveItem.ID
        inode.Unlock()

        return f.InsertID(id, inode), nil
}

// MoveID moves an item to a new ID name. Also responsible for handling the
// actual overwrite of the item's IDInternal field
func (f *Filesystem) MoveID(oldID string, newID string) error {
        inode := f.GetID(oldID)
        if inode == nil {
                // It may have already been renamed. This is not an error. We assume
                // that IDs will never collide. Re-perform the op if this is the case.
                if inode = f.GetID(newID); inode == nil {
                        // nope, it just doesn't exist
                        return errors.New("Could not get item: " + oldID)
                }
        }

        // need to rename the child under the parent
        parent := f.GetID(inode.ParentID())
        parent.Lock()
        for i, child := range parent.children {
                if child == oldID {
                        parent.children[i] = newID
                        break
                }
        }
        parent.Unlock()

        // now actually perform the metadata+content move
        f.DeleteID(oldID)
        f.InsertID(newID, inode)
        if inode.IsDir() {
                return nil
        }
        f.content.Move(oldID, newID)
        return nil
}

// MovePath moves an item to a new position.
func (f *Filesystem) MovePath(oldParent, newParent, oldName, newName string, auth *graph.Auth) error {
        inode, err := f.GetChild(oldParent, oldName, auth)
        if err != nil {
                return err
        }

        id := inode.ID()
        f.DeleteID(id)

        // this is the actual move op
        inode.SetName(newName)
        parent := f.GetID(newParent)
        inode.Parent.ID = parent.DriveItem.ID
        f.InsertID(id, inode)
        return nil
}

// SerializeAll dumps all inode metadata currently in the cache to disk. This
// metadata is only used later if an item could not be found in memory AND the
// cache is offline. Old metadata is not removed, only overwritten (to avoid an
// offline session from wiping all metadata on a subsequent serialization).
func (f *Filesystem) SerializeAll() {
        log.Debug().Msg("Serializing cache metadata to disk.")

        allItems := make(map[string][]byte)
        f.metadata.Range(func(k interface{}, v interface{}) bool {
                // cannot occur within bolt transaction because acquiring the inode lock
                // with AsJSON locks out other boltdb transactions
                id := fmt.Sprint(k)
                allItems[id] = v.(*Inode).AsJSON()
                return true
        })

        /*
                One transaction to serialize them all,
                One transaction to find them,
                One transaction to bring them all
                and in the darkness write them.
        */
        f.db.Batch(func(tx *bolt.Tx) error {
                b := tx.Bucket(bucketMetadata)
                for k, v := range allItems {
                        b.Put([]byte(k), v)
                        if k == f.root {
                                // root item must be updated manually (since there's actually
                                // two copies)
                                b.Put([]byte("root"), v)
                        }
                }
                return nil
        })
}

1	package fs
2
3	import (
4	"errors"
5	"fmt"
6	"os"
7	"path/filepath"
8	"strings"
9	"sync"
10	"time"
11
12	"github.com/hanwen/go-fuse/v2/fuse"
13	"github.com/jstaf/onedriver/fs/graph"
14	"github.com/rs/zerolog/log"
15	bolt "go.etcd.io/bbolt"
16	)
17
18	// Filesystem is the actual FUSE filesystem and uses the go analogy of the
19	// "low-level" FUSE API here:
20	// https://github.com/libfuse/libfuse/blob/master/include/fuse_lowlevel.h
21	type Filesystem struct {
22	fuse.RawFileSystem
23
24	metadata sync.Map
25	db *bolt.DB
26	content *LoopbackCache
27	auth *graph.Auth
28	root string // the id of the filesystem's root item
29	deltaLink string
30	uploads *UploadManager
31
32	sync.RWMutex
33	offline bool
34	lastNodeID uint64
35	inodes []string
36
37	// tracks currently open directories
38	opendirsM sync.RWMutex
39	opendirs map[uint64][]*Inode
40	}
41
42	// boltdb buckets
43	var (
44	bucketContent = []byte("content")
45	bucketMetadata = []byte("metadata")
46	bucketDelta = []byte("delta")
47	bucketVersion = []byte("version")
48	)
49
50	// so we can tell what format the db has
51	const fsVersion = "1"
52
53	// NewFilesystem creates a new filesystem
54	func NewFilesystem(auth graph.Auth, cacheDir string) Filesystem {	9✔
55	// prepare cache directory	9✔
56	if _, err := os.Stat(cacheDir); err != nil {	18✔
57	if err = os.Mkdir(cacheDir, 0700); err != nil {	9✔
58	log.Fatal().Err(err).Msg("Could not create cache directory.")	×
59	}	×
60	}
61	db, err := bolt.Open(	9✔
62	filepath.Join(cacheDir, "onedriver.db"),	9✔
63	0600,	9✔
64	&bolt.Options{Timeout: time.Second * 5},	9✔
65	)	9✔
66	if err != nil {	9✔
67	log.Fatal().Err(err).Msg("Could not open DB. Is it already in use by another mount?")	×
68	}	×
69
70	content := NewLoopbackCache(filepath.Join(cacheDir, "content"))	9✔
71	db.Update(func(tx *bolt.Tx) error {	18✔
72	tx.CreateBucketIfNotExists(bucketMetadata)	9✔
73	tx.CreateBucketIfNotExists(bucketDelta)	9✔
74	versionBucket, _ := tx.CreateBucketIfNotExists(bucketVersion)	9✔
75		9✔
76	// migrate old content bucket to the local filesystem	9✔
77	b := tx.Bucket(bucketContent)	9✔
78	if b != nil {	9✔
79	oldVersion := "0"	×
80	log.Info().	×
81	Str("oldVersion", oldVersion).	×
82	Str("version", fsVersion).	×
83	Msg("Migrating to new db format.")	×
84	err := b.ForEach(func(k []byte, v []byte) error {	×
85	log.Info().Bytes("key", k).Msg("Migrating file content.")	×
86	if err := content.Insert(string(k), v); err != nil {	×
87	return err	×
88	}	×
89	return b.Delete(k)	×
90	})
91	if err != nil {	×
92	log.Error().Err(err).Msg("Migration failed.")	×
93	}	×
94	tx.DeleteBucket(bucketContent)	×
95	log.Info().	×
96	Str("oldVersion", oldVersion).	×
97	Str("version", fsVersion).	×
98	Msg("Migrations complete.")	×
99	}
100	return versionBucket.Put([]byte("version"), []byte(fsVersion))	9✔
101	})
102
103	// ok, ready to start fs
104	fs := &Filesystem{	9✔
105	RawFileSystem: fuse.NewDefaultRawFileSystem(),	9✔
106	content: content,	9✔
107	db: db,	9✔
108	auth: auth,	9✔
109	opendirs: make(map[uint64][]*Inode),	9✔
110	}	9✔
111		9✔
112	rootItem, err := graph.GetItem("root", auth)	9✔
113	root := NewInodeDriveItem(rootItem)	9✔
114	if err != nil {	9✔
115	if graph.IsOffline(err) {	×
116	// no network, load from db if possible and go to read-only state	×
117	fs.Lock()	×
118	fs.offline = true	×
119	fs.Unlock()	×
120	if root = fs.GetID("root"); root == nil {	×
121	log.Fatal().Msg(	×
122	"We are offline and could not fetch the filesystem root item from disk.",	×
123	)	×
124	}	×
125	// when offline, we load the cache deltaLink from disk
126	fs.db.View(func(tx *bolt.Tx) error {	×
127	if link := tx.Bucket(bucketDelta).Get([]byte("deltaLink")); link != nil {	×
128	fs.deltaLink = string(link)	×
129	} else {	×
130	// Only reached if a previous online session never survived	×
131	// long enough to save its delta link. We explicitly disallow these	×
132	// types of startups as it's possible for things to get out of sync	×
133	// this way.	×
134	log.Fatal().Msg("Cannot perform an offline startup without a valid " +	×
135	"delta link from a previous session.")	×
136	}	×
137	return nil	×
138	})
139	} else {	×
140	log.Fatal().Err(err).Msg("Could not fetch root item of filesystem!")	×
141	}	×
142	}
143	// root inode is inode 1
144	fs.root = root.ID()	9✔
145	fs.InsertID(fs.root, root)	9✔
146		9✔
147	fs.uploads = NewUploadManager(2*time.Second, db, fs, auth)	9✔
148		9✔
149	if !fs.IsOffline() {	18✔
150	// .Trash-UID is used by "gio trash" for user trash, create it if it	9✔
151	// does not exist	9✔
152	trash := fmt.Sprintf(".Trash-%d", os.Getuid())	9✔
153	if child, _ := fs.GetChild(fs.root, trash, auth); child == nil {	9✔
154	item, err := graph.Mkdir(trash, fs.root, auth)	×
155	if err != nil {	×
156	log.Error().Err(err).	×
157	Msg("Could not create trash folder. " +	×
158	"Trashing items through the file browser may result in errors.")	×
159	} else {	×
160	fs.InsertID(item.ID, NewInodeDriveItem(item))	×
161	}	×
162	}
163
164	// using token=latest because we don't care about existing items - they'll
165	// be downloaded on-demand by the cache
166	fs.deltaLink = "/me/drive/root/delta?token=latest"	9✔
167	}
168
169	// deltaloop is started manually
170	return fs	9✔
171	}
172
173	// IsOffline returns whether or not the cache thinks its offline.
174	func (f *Filesystem) IsOffline() bool {	104✔
175	f.RLock()	104✔
176	defer f.RUnlock()	104✔
177	return f.offline	104✔
178	}	104✔
179
180	// TranslateID returns the DriveItemID for a given NodeID
181	func (f *Filesystem) TranslateID(nodeID uint64) string {	2,083✔
182	f.RLock()	2,083✔
183	defer f.RUnlock()	2,083✔
184	if nodeID > f.lastNodeID \|\| nodeID == 0 {	2,083✔
185	return ""	×
186	}	×
187	return f.inodes[nodeID-1]	2,083✔
188	}
189
190	// GetNodeID fetches the inode for a particular inode ID.
191	func (f Filesystem) GetNodeID(nodeID uint64) Inode {	332✔
192	id := f.TranslateID(nodeID)	332✔
193	if id == "" {	332✔
194	return nil	×
195	}	×
196	return f.GetID(id)	332✔
197	}
198
199	// InsertNodeID assigns a numeric inode ID used by the kernel if one is not
200	// already assigned.
201	func (f Filesystem) InsertNodeID(inode Inode) uint64 {	1,087✔
202	nodeID := inode.NodeID()	1,087✔
203	if nodeID == 0 {	2,133✔
204	// lock ordering is to satisfy deadlock detector	1,046✔
205	inode.Lock()	1,046✔
206	f.Lock()	1,046✔
207		1,046✔
208	f.lastNodeID++	1,046✔
209	f.inodes = append(f.inodes, inode.DriveItem.ID)	1,046✔
210	nodeID = f.lastNodeID	1,046✔
211	inode.nodeID = nodeID	1,046✔
212		1,046✔
213	f.Unlock()	1,046✔
214	inode.Unlock()	1,046✔
215	}	1,046✔
216	return nodeID	1,087✔
217	}
218
219	// GetID gets an inode from the cache by ID. No API fetching is performed.
220	// Result is nil if no inode is found.
221	func (f Filesystem) GetID(id string) Inode {	111,648✔
222	entry, exists := f.metadata.Load(id)	111,648✔
223	if !exists {	111,684✔
224	// we allow fetching from disk as a fallback while offline (and it's also	36✔
225	// necessary while transitioning from offline->online)	36✔
226	var found *Inode	36✔
227	f.db.View(func(tx *bolt.Tx) error {	72✔
228	data := tx.Bucket(bucketMetadata).Get([]byte(id))	36✔
229	var err error	36✔
230	if data != nil {	36✔
231	found, err = NewInodeJSON(data)	×
232	}	×
233	return err	36✔
234	})
235	if found != nil {	36✔
236	f.InsertNodeID(found)	×
237	f.metadata.Store(id, found) // move to memory for next time	×
238	}	×
239	return found	36✔
240	}
241	return entry.(*Inode)	111,612✔
242	}
243
244	// InsertID inserts a single item into the filesystem by ID and sets its parent
245	// using the Inode.Parent.ID, if set. Must be called after DeleteID, if being
246	// used to rename/move an item. This is the main way new Inodes are added to the
247	// filesystem. Returns the Inode's numeric NodeID.
248	func (f Filesystem) InsertID(id string, inode Inode) uint64 {	109✔
249	f.metadata.Store(id, inode)	109✔
250	nodeID := f.InsertNodeID(inode)	109✔
251		109✔
252	if id != inode.ID() {	140✔
253	// we update the inode IDs here in case they do not match/changed	31✔
254	inode.Lock()	31✔
255	inode.DriveItem.ID = id	31✔
256	inode.Unlock()	31✔
257		31✔
258	f.Lock()	31✔
259	if nodeID <= f.lastNodeID {	62✔
260	f.inodes[nodeID-1] = id	31✔
261	} else {	31✔
262	log.Error().	×
263	Uint64("nodeID", nodeID).	×
264	Uint64("lastNodeID", f.lastNodeID).	×
265	Msg("NodeID exceeded maximum node ID! Ignoring ID change.")	×
266	}	×
267	f.Unlock()	31✔
268	}
269
270	parentID := inode.ParentID()	109✔
271	if parentID == "" {	118✔
272	// root item, or parent not set	9✔
273	return nodeID	9✔
274	}	9✔
275	parent := f.GetID(parentID)	100✔
276	if parent == nil {	100✔
277	log.Error().	×
278	Str("parentID", parentID).	×
279	Str("childID", id).	×
280	Str("childName", inode.Name()).	×
281	Msg("Parent item could not be found when setting parent.")	×
282	return nodeID	×
283	}	×
284
285	// check if the item has already been added to the parent
286	// Lock order is super key here, must go parent->child or the deadlock
287	// detector screams at us.
288	parent.Lock()	100✔
289	defer parent.Unlock()	100✔
290	for _, child := range parent.children {	3,088✔
291	if child == id {	3,020✔
292	// exit early, child cannot be added twice	32✔
293	return nodeID	32✔
294	}	32✔
295	}
296
297	// add to parent
298	if inode.IsDir() {	82✔
299	parent.subdir++	14✔
300	}	14✔
301	parent.children = append(parent.children, id)	68✔
302		68✔
303	return nodeID	68✔
304	}
305
306	// InsertChild adds an item as a child of a specified parent ID.
307	func (f Filesystem) InsertChild(parentID string, child Inode) uint64 {	55✔
308	child.Lock()	55✔
309	// should already be set, just double-checking here.	55✔
310	child.DriveItem.Parent.ID = parentID	55✔
311	id := child.DriveItem.ID	55✔
312	child.Unlock()	55✔
313	return f.InsertID(id, child)	55✔
314	}	55✔
315
316	// DeleteID deletes an item from the cache, and removes it from its parent. Must
317	// be called before InsertID if being used to rename/move an item.
318	func (f *Filesystem) DeleteID(id string) {	56✔
319	if inode := f.GetID(id); inode != nil {	109✔
320	parent := f.GetID(inode.ParentID())	53✔
321	parent.Lock()	53✔
322	for i, childID := range parent.children {	1,542✔
323	if childID == id {	1,511✔
324	parent.children = append(parent.children[:i], parent.children[i+1:]...)	22✔
325	if inode.IsDir() {	30✔
326	parent.subdir--	8✔
327	}	8✔
328	break	22✔
329	}
330	}
331	parent.Unlock()	53✔
332	}
333	f.metadata.Delete(id)	56✔
334	f.uploads.CancelUpload(id)	56✔
335	}
336
337	// GetChild fetches a named child of an item. Wraps GetChildrenID.
338	func (f Filesystem) GetChild(id string, name string, auth graph.Auth) (*Inode, error) {	692✔
339	children, err := f.GetChildrenID(id, auth)	692✔
340	if err != nil {	692✔
341	return nil, err	×
342	}	×
343	for _, child := range children {	57,350✔
344	if strings.EqualFold(child.Name(), name) {	57,155✔
345	return child, nil	497✔
346	}	497✔
347	}
348	return nil, errors.New("child does not exist")	195✔
349	}
350
351	// GetChildrenID grabs all DriveItems that are the children of the given ID. If
352	// items are not found, they are fetched.
353	func (f Filesystem) GetChildrenID(id string, auth graph.Auth) (map[string]*Inode, error) {	778✔
354	// fetch item and catch common errors	778✔
355	inode := f.GetID(id)	778✔
356	children := make(map[string]*Inode)	778✔
357	if inode == nil {	778✔
358	log.Error().Str("id", id).Msg("Inode not found in cache")	×
359	return children, errors.New(id + " not found in cache")	×
360	} else if !inode.IsDir() {	778✔
361	// Normal files are treated as empty folders. This only gets called if	×
362	// we messed up and tried to get the children of a plain-old file.	×
363	log.Warn().	×
364	Str("id", id).	×
365	Str("path", inode.Path()).	×
366	Msg("Attepted to get children of ordinary file")	×
367	return children, nil	×
368	}	×
369
370	// If item.children is not nil, it means we have the item's children
371	// already and can fetch them directly from the cache
372	inode.RLock()	778✔
373	if inode.children != nil {	1,536✔
374	// can potentially have out-of-date child metadata if started offline, but since	758✔
375	// changes are disallowed while offline, the children will be back in sync after	758✔
376	// the first successful delta fetch (which also brings the fs back online)	758✔
377	for _, childID := range inode.children {	109,598✔
378	child := f.GetID(childID)	108,840✔
379	if child == nil {	108,840✔
380	// will be nil if deleted or never existed	×
381	continue	×
382	}
383	children[strings.ToLower(child.Name())] = child	108,840✔
384	}
385	inode.RUnlock()	758✔
386	return children, nil	758✔
387	}
388	inode.RUnlock()	20✔
389		20✔
390	// We haven't fetched the children for this item yet, get them from the server.	20✔
391	fetched, err := graph.GetItemChildren(id, auth)	20✔
392	if err != nil {	20✔
393	if graph.IsOffline(err) {	×
394	log.Warn().Str("id", id).	×
395	Msg("We are offline, and no children found in cache. " +	×
396	"Pretending there are no children.")	×
397	return children, nil	×
398	}	×
399	// something else happened besides being offline
400	return nil, err	×
401	}
402
403	inode.Lock()	20✔
404	inode.children = make([]string, 0)	20✔
405	for _, item := range fetched {	998✔
406	// we will always have an id after fetching from the server	978✔
407	child := NewInodeDriveItem(item)	978✔
408	f.InsertNodeID(child)	978✔
409	f.metadata.Store(child.DriveItem.ID, child)	978✔
410		978✔
411	// store in result map	978✔
412	children[strings.ToLower(child.Name())] = child	978✔
413		978✔
414	// store id in parent item and increment parents subdirectory count	978✔
415	inode.children = append(inode.children, child.DriveItem.ID)	978✔
416	if child.IsDir() {	1,035✔
417	inode.subdir++	57✔
418	}	57✔
419	}
420	inode.Unlock()	20✔
421		20✔
422	return children, nil	20✔
423	}
424
425	// GetChildrenPath grabs all DriveItems that are the children of the resource at
426	// the path. If items are not found, they are fetched.
427	func (f Filesystem) GetChildrenPath(path string, auth graph.Auth) (map[string]*Inode, error) {	2✔
428	inode, err := f.GetPath(path, auth)	2✔
429	if err != nil {	2✔
430	return make(map[string]*Inode), err	×
431	}	×
432	return f.GetChildrenID(inode.ID(), auth)	2✔
433	}
434
435	// GetPath fetches a given DriveItem in the cache, if any items along the way are
436	// not found, they are fetched.
437	func (f Filesystem) GetPath(path string, auth graph.Auth) (*Inode, error) {	16✔
438	lastID := f.root	16✔
439	if path == "/" {	21✔
440	return f.GetID(lastID), nil	5✔
441	}	5✔
442
443	// from the root directory, traverse the chain of items till we reach our
444	// target ID.
445	path = strings.TrimSuffix(strings.ToLower(path), "/")	11✔
446	split := strings.Split(path, "/")[1:] //omit leading "/"	11✔
447	var inode *Inode	11✔
448	for i := 0; i < len(split); i++ {	26✔
449	// fetches children	15✔
450	children, err := f.GetChildrenID(lastID, auth)	15✔
451	if err != nil {	15✔
452	return nil, err	×
453	}	×
454
455	var exists bool // if we use ":=", item is shadowed	15✔
456	inode, exists = children[split[i]]	15✔
457	if !exists {	15✔
458	// the item still doesn't exist after fetching from server. it	×
459	// doesn't exist	×
460	return nil, errors.New(strings.Join(split[:i+1], "/") +	×
461	" does not exist on server or in local cache")	×
462	}	×
463	lastID = inode.ID()	15✔
464	}
465	return inode, nil	11✔
466	}
467
468	// DeletePath an item from the cache by path. Must be called before Insert if
469	// being used to move/rename an item.
470	func (f *Filesystem) DeletePath(key string) {	1✔
471	inode, _ := f.GetPath(strings.ToLower(key), nil)	1✔
472	if inode != nil {	2✔
473	f.DeleteID(inode.ID())	1✔
474	}	1✔
475	}
476
477	// InsertPath lets us manually insert an item to the cache (like if it was
478	// created locally). Overwrites a cached item if present. Must be called after
479	// delete if being used to move/rename an item.
480	func (f Filesystem) InsertPath(key string, auth graph.Auth, inode *Inode) (uint64, error) {	4✔
481	key = strings.ToLower(key)	4✔
482		4✔
483	// set the item.Parent.ID properly if the item hasn't been in the cache	4✔
484	// before or is being moved.	4✔
485	parent, err := f.GetPath(filepath.Dir(key), auth)	4✔
486	if err != nil {	4✔
487	return 0, err	×
488	} else if parent == nil {	4✔
489	const errMsg string = "parent of key was nil"	×
490	log.Error().	×
491	Str("key", key).	×
492	Str("path", inode.Path()).	×
493	Msg(errMsg)	×
494	return 0, errors.New(errMsg)	×
495	}	×
496
497	// Coded this way to make sure locks are in the same order for the deadlock
498	// detector (lock ordering needs to be the same as InsertID: Parent->Child).
499	parentID := parent.ID()	4✔
500	inode.Lock()	4✔
501	inode.DriveItem.Parent.ID = parentID	4✔
502	id := inode.DriveItem.ID	4✔
503	inode.Unlock()	4✔
504		4✔
505	return f.InsertID(id, inode), nil	4✔
506	}
507
508	// MoveID moves an item to a new ID name. Also responsible for handling the
509	// actual overwrite of the item's IDInternal field
510	func (f *Filesystem) MoveID(oldID string, newID string) error {	32✔
511	inode := f.GetID(oldID)	32✔
512	if inode == nil {	33✔
513	// It may have already been renamed. This is not an error. We assume	1✔
514	// that IDs will never collide. Re-perform the op if this is the case.	1✔
515	if inode = f.GetID(newID); inode == nil {	1✔
516	// nope, it just doesn't exist	×
517	return errors.New("Could not get item: " + oldID)	×
518	}	×
519	}
520
521	// need to rename the child under the parent
522	parent := f.GetID(inode.ParentID())	32✔
523	parent.Lock()	32✔
524	for i, child := range parent.children {	879✔
525	if child == oldID {	878✔
526	parent.children[i] = newID	31✔
527	break	31✔
528	}
529	}
530	parent.Unlock()	32✔
531		32✔
532	// now actually perform the metadata+content move	32✔
533	f.DeleteID(oldID)	32✔
534	f.InsertID(newID, inode)	32✔
535	if inode.IsDir() {	32✔
536	return nil	×
537	}	×
538	f.content.Move(oldID, newID)	32✔
539	return nil	32✔
540	}
541
542	// MovePath moves an item to a new position.
543	func (f Filesystem) MovePath(oldParent, newParent, oldName, newName string, auth graph.Auth) error {	9✔
544	inode, err := f.GetChild(oldParent, oldName, auth)	9✔
545	if err != nil {	9✔
546	return err	×
547	}	×
548
549	id := inode.ID()	9✔
550	f.DeleteID(id)	9✔
551		9✔
552	// this is the actual move op	9✔
553	inode.SetName(newName)	9✔
554	parent := f.GetID(newParent)	9✔
555	inode.Parent.ID = parent.DriveItem.ID	9✔
556	f.InsertID(id, inode)	9✔
557	return nil	9✔
558	}
559
560	// SerializeAll dumps all inode metadata currently in the cache to disk. This
561	// metadata is only used later if an item could not be found in memory AND the
562	// cache is offline. Old metadata is not removed, only overwritten (to avoid an
563	// offline session from wiping all metadata on a subsequent serialization).
564	func (f *Filesystem) SerializeAll() {	28✔
565	log.Debug().Msg("Serializing cache metadata to disk.")	28✔
566		28✔
567	allItems := make(map[string][]byte)	28✔
568	f.metadata.Range(func(k interface{}, v interface{}) bool {	10,827✔
569	// cannot occur within bolt transaction because acquiring the inode lock	10,799✔
570	// with AsJSON locks out other boltdb transactions	10,799✔
571	id := fmt.Sprint(k)	10,799✔
572	allItems[id] = v.(*Inode).AsJSON()	10,799✔
573	return true	10,799✔
574	})	10,799✔
575
576	/*
577	One transaction to serialize them all,
578	One transaction to find them,
579	One transaction to bring them all
580	and in the darkness write them.
581	*/
582	f.db.Batch(func(tx *bolt.Tx) error {	56✔
583	b := tx.Bucket(bucketMetadata)	28✔
584	for k, v := range allItems {	10,827✔
585	b.Put([]byte(k), v)	10,799✔
586	if k == f.root {	10,827✔
587	// root item must be updated manually (since there's actually	28✔
588	// two copies)	28✔
589	b.Put([]byte("root"), v)	28✔
590	}	28✔
591	}
592	return nil	28✔
593	})
594	}

jstaf / onedriver / 6539229776

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous