6554536784

Committed 18 Oct 2023 12:37AM UTC coverage: 74.287% (+0.5%) from 73.744%

Build # 6554536784

Build Type

Pull #359

github

Committed by

jstaf

Commit Message

fix hash stream functions

Pull Request Pull Request #359: Fix file redownloads, Ubuntu 20/Debian 11 compatibility

Run Details

13 of 17 new or added lines in 3 files covered. (76.47%)

13 existing lines in 3 files now uncovered.

1901 of 2559 relevant lines covered (74.29%)

341244.46 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

85.51

/fs/graph/drive_item.go

package graph

import (
        "bytes"
        "encoding/json"
        "fmt"
        "io"
        "net/url"
        "strings"
        "time"

        "github.com/rs/zerolog/log"
)

// DriveTypePersonal and friends represent the possible different values for a
// drive's type when fetched from the API.
const (
        DriveTypePersonal   = "personal"
        DriveTypeBusiness   = "business"
        DriveTypeSharepoint = "documentLibrary"
)

// DriveItemParent describes a DriveItem's parent in the Graph API
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/resources/itemreference
type DriveItemParent struct {
        //TODO Path is technically available, but we shouldn't use it
        Path      string `json:"path,omitempty"`
        ID        string `json:"id,omitempty"`
        DriveID   string `json:"driveId,omitempty"`
        DriveType string `json:"driveType,omitempty"` // personal | business | documentLibrary
}

// Folder is used for parsing only
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/resources/folder
type Folder struct {
        ChildCount uint32 `json:"childCount,omitempty"`
}

// Hashes are integrity hashes used to determine if file content has changed.
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/resources/hashes
type Hashes struct {
        SHA1Hash     string `json:"sha1Hash,omitempty"`
        QuickXorHash string `json:"quickXorHash,omitempty"`
}

// File is used for checking for changes in local files (relative to the server).
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/resources/file
type File struct {
        Hashes Hashes `json:"hashes,omitempty"`
}

// Deleted is used for detecting when items get deleted on the server
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/resources/deleted
type Deleted struct {
        State string `json:"state,omitempty"`
}

// DriveItem contains the data fields from the Graph API
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/resources/driveitem
type DriveItem struct {
        ID               string           `json:"id,omitempty"`
        Name             string           `json:"name,omitempty"`
        Size             uint64           `json:"size,omitempty"`
        ModTime          *time.Time       `json:"lastModifiedDatetime,omitempty"`
        Parent           *DriveItemParent `json:"parentReference,omitempty"`
        Folder           *Folder          `json:"folder,omitempty"`
        File             *File            `json:"file,omitempty"`
        Deleted          *Deleted         `json:"deleted,omitempty"`
        ConflictBehavior string           `json:"@microsoft.graph.conflictBehavior,omitempty"`
        ETag             string           `json:"eTag,omitempty"`
}

// IsDir returns if the DriveItem represents a directory or not
func (d *DriveItem) IsDir() bool {
        return d.Folder != nil
}

// ModTimeUnix returns the modification time as a unix uint64 time
func (d *DriveItem) ModTimeUnix() uint64 {
        return uint64(d.ModTime.Unix())
}

// getItem is the internal method used to lookup items
func getItem(path string, auth *Auth) (*DriveItem, error) {
        body, err := Get(path, auth)
        if err != nil {
                return nil, err
        }
        item := &DriveItem{}
        err = json.Unmarshal(body, item)
        if err != nil && bytes.Contains(body, []byte("\"size\":-")) {
                // onedrive for business directories can sometimes have negative sizes,
                // ignore this error
                err = nil
        }
        return item, err
}

// GetItem fetches a DriveItem by ID. ID can also be "root" for the root item.
func GetItem(id string, auth *Auth) (*DriveItem, error) {
        return getItem(IDPath(id), auth)
}

// GetItemChild fetches the named child of an item.
func GetItemChild(id string, name string, auth *Auth) (*DriveItem, error) {
        return getItem(
                fmt.Sprintf("%s:/%s", IDPath(id), url.PathEscape(name)),
                auth,
        )
}

// GetItemPath fetches a DriveItem by path. Only used in special cases, like for the
// root item.
func GetItemPath(path string, auth *Auth) (*DriveItem, error) {
        return getItem(ResourcePath(path), auth)
}

// GetItemContent retrieves an item's content from the Graph endpoint.
func GetItemContent(id string, auth *Auth) ([]byte, uint64, error) {
        buf := bytes.NewBuffer(make([]byte, 0))
        n, err := GetItemContentStream(id, auth, buf)
        return buf.Bytes(), uint64(n), err
}

// GetItemContentStream is the same as GetItemContent, but writes data to an
// output reader. This function assumes a brand-new io.Writer is used, so
// "output" must be truncated if there is content already in the io.Writer
// prior to use.
func GetItemContentStream(id string, auth *Auth, output io.Writer) (uint64, error) {
        // determine the size of the item
        item, err := GetItem(id, auth)
        if err != nil {
                return 0, err
        }

        const downloadChunkSize = 10 * 1024 * 1024
        downloadURL := fmt.Sprintf("/me/drive/items/%s/content", id)
        if item.Size <= downloadChunkSize {
                // simple one-shot download
                content, err := Get(downloadURL, auth)
                if err != nil {
                        return 0, err
                }
                n, err := output.Write(content)
                return uint64(n), err
        }

        // multipart download
        var n uint64
        for i := 0; i < int(item.Size/downloadChunkSize)+1; i++ {
                start := i * downloadChunkSize
                end := start + downloadChunkSize - 1
                log.Info().
                        Str("id", item.ID).
                        Str("name", item.Name).
                        Msgf("Downloading bytes %d-%d/%d.", start, end, item.Size)
                content, err := Get(downloadURL, auth, Header{
                        key:   "Range",
                        value: fmt.Sprintf("bytes=%d-%d", start, end),
                })
                if err != nil {
                        return n, err
                }
                written, err := output.Write(content)
                n += uint64(written)
                if err != nil {
                        return n, err
                }
        }
        log.Info().
                Str("id", item.ID).
                Str("name", item.Name).
                Uint64("size", n).
                Msgf("Download completed!")
        return n, nil
}

// Remove removes a directory or file by ID
func Remove(id string, auth *Auth) error {
        return Delete("/me/drive/items/"+id, auth)
}

// Mkdir creates a directory on the server at the specified parent ID.
func Mkdir(name string, parentID string, auth *Auth) (*DriveItem, error) {
        // create a new folder on the server
        newFolderPost := DriveItem{
                Name:   name,
                Folder: &Folder{},
        }
        bytePayload, _ := json.Marshal(newFolderPost)
        resp, err := Post(childrenPathID(parentID), auth, bytes.NewReader(bytePayload))
        if err != nil {
                return nil, err
        }
        err = json.Unmarshal(resp, &newFolderPost)
        return &newFolderPost, err
}

// Rename moves and/or renames an item on the server. The itemName and parentID
// arguments correspond to the *new* basename or id of the parent.
func Rename(itemID string, itemName string, parentID string, auth *Auth) error {
        // start creating patch content for server
        // mutex does not need to be initialized since it is never used locally
        patchContent := DriveItem{
                ConflictBehavior: "replace", // overwrite existing content at new location
                Name:             itemName,
                Parent: &DriveItemParent{
                        ID: parentID,
                },
        }

        // apply patch to server copy - note that we don't actually care about the
        // response content, only if it returns an error
        jsonPatch, _ := json.Marshal(patchContent)
        _, err := Patch("/me/drive/items/"+itemID, auth, bytes.NewReader(jsonPatch))
        if err != nil && strings.Contains(err.Error(), "resourceModified") {
                // Wait a second, then retry the request. The Onedrive servers sometimes
                // aren't quick enough here if the object has been recently created
                // (<1 second ago).
                time.Sleep(time.Second)
                _, err = Patch("/me/drive/items/"+itemID, auth, bytes.NewReader(jsonPatch))
        }
        return err
}

// only used for parsing
type driveChildren struct {
        Children []*DriveItem `json:"value"`
        NextLink string       `json:"@odata.nextLink"`
}

// this is the internal method that actually fetches an item's children
func getItemChildren(pollURL string, auth *Auth) ([]*DriveItem, error) {
        fetched := make([]*DriveItem, 0)
        for pollURL != "" {
                body, err := Get(pollURL, auth)
                if err != nil {
                        return fetched, err
                }
                var pollResult driveChildren
                json.Unmarshal(body, &pollResult)

                // there can be multiple pages of 200 items each (default).
                // continue to next interation if we have an @odata.nextLink value
                fetched = append(fetched, pollResult.Children...)
                pollURL = strings.TrimPrefix(pollResult.NextLink, GraphURL)
        }
        return fetched, nil
}

// GetItemChildren fetches all children of an item denoted by ID.
func GetItemChildren(id string, auth *Auth) ([]*DriveItem, error) {
        return getItemChildren(childrenPathID(id), auth)
}

// GetItemChildrenPath fetches all children of an item denoted by path.
func GetItemChildrenPath(path string, auth *Auth) ([]*DriveItem, error) {
        return getItemChildren(childrenPath(path), auth)
}

1	package graph
2
3	import (
4	"bytes"
5	"encoding/json"
6	"fmt"
7	"io"
8	"net/url"
9	"strings"
10	"time"
11
12	"github.com/rs/zerolog/log"
13	)
14
15	// DriveTypePersonal and friends represent the possible different values for a
16	// drive's type when fetched from the API.
17	const (
18	DriveTypePersonal = "personal"
19	DriveTypeBusiness = "business"
20	DriveTypeSharepoint = "documentLibrary"
21	)
22
23	// DriveItemParent describes a DriveItem's parent in the Graph API
24	// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/resources/itemreference
25	type DriveItemParent struct {
26	//TODO Path is technically available, but we shouldn't use it
27	Path string `json:"path,omitempty"`
28	ID string `json:"id,omitempty"`
29	DriveID string `json:"driveId,omitempty"`
30	DriveType string `json:"driveType,omitempty"` // personal \| business \| documentLibrary
31	}
32
33	// Folder is used for parsing only
34	// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/resources/folder
35	type Folder struct {
36	ChildCount uint32 `json:"childCount,omitempty"`
37	}
38
39	// Hashes are integrity hashes used to determine if file content has changed.
40	// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/resources/hashes
41	type Hashes struct {
42	SHA1Hash string `json:"sha1Hash,omitempty"`
43	QuickXorHash string `json:"quickXorHash,omitempty"`
44	}
45
46	// File is used for checking for changes in local files (relative to the server).
47	// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/resources/file
48	type File struct {
49	Hashes Hashes `json:"hashes,omitempty"`
50	}
51
52	// Deleted is used for detecting when items get deleted on the server
53	// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/resources/deleted
54	type Deleted struct {
55	State string `json:"state,omitempty"`
56	}
57
58	// DriveItem contains the data fields from the Graph API
59	// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/resources/driveitem
60	type DriveItem struct {
61	ID string `json:"id,omitempty"`
62	Name string `json:"name,omitempty"`
63	Size uint64 `json:"size,omitempty"`
64	ModTime *time.Time `json:"lastModifiedDatetime,omitempty"`
65	Parent *DriveItemParent `json:"parentReference,omitempty"`
66	Folder *Folder `json:"folder,omitempty"`
67	File *File `json:"file,omitempty"`
68	Deleted *Deleted `json:"deleted,omitempty"`
69	ConflictBehavior string `json:"@microsoft.graph.conflictBehavior,omitempty"`
70	ETag string `json:"eTag,omitempty"`
71	}
72
73	// IsDir returns if the DriveItem represents a directory or not
74	func (d *DriveItem) IsDir() bool {	5,390✔
75	return d.Folder != nil	5,390✔
76	}	5,390✔
77
78	// ModTimeUnix returns the modification time as a unix uint64 time
79	func (d *DriveItem) ModTimeUnix() uint64 {	1,830✔
80	return uint64(d.ModTime.Unix())	1,830✔
81	}	1,830✔
82
83	// getItem is the internal method used to lookup items
84	func getItem(path string, auth Auth) (DriveItem, error) {	112✔
85	body, err := Get(path, auth)	112✔
86	if err != nil {	133✔
87	return nil, err	21✔
88	}	21✔
89	item := &DriveItem{}	91✔
90	err = json.Unmarshal(body, item)	91✔
91	if err != nil && bytes.Contains(body, []byte("\"size\":-")) {	91✔
UNCOV 92	// onedrive for business directories can sometimes have negative sizes,	×
UNCOV 93	// ignore this error	×
UNCOV 94	err = nil	×
UNCOV 95	}	×
96	return item, err	91✔
97	}
98
99	// GetItem fetches a DriveItem by ID. ID can also be "root" for the root item.
100	func GetItem(id string, auth Auth) (DriveItem, error) {	44✔
101	return getItem(IDPath(id), auth)	44✔
102	}	44✔
103
104	// GetItemChild fetches the named child of an item.
105	func GetItemChild(id string, name string, auth Auth) (DriveItem, error) {	5✔
106	return getItem(	5✔
107	fmt.Sprintf("%s:/%s", IDPath(id), url.PathEscape(name)),	5✔
108	auth,	5✔
109	)	5✔
110	}	5✔
111
112	// GetItemPath fetches a DriveItem by path. Only used in special cases, like for the
113	// root item.
114	func GetItemPath(path string, auth Auth) (DriveItem, error) {	63✔
115	return getItem(ResourcePath(path), auth)	63✔
116	}	63✔
117
118	// GetItemContent retrieves an item's content from the Graph endpoint.
119	func GetItemContent(id string, auth *Auth) ([]byte, uint64, error) {	22✔
120	buf := bytes.NewBuffer(make([]byte, 0))	22✔
121	n, err := GetItemContentStream(id, auth, buf)	22✔
122	return buf.Bytes(), uint64(n), err	22✔
123	}	22✔
124
125	// GetItemContentStream is the same as GetItemContent, but writes data to an
126	// output reader. This function assumes a brand-new io.Writer is used, so
127	// "output" must be truncated if there is content already in the io.Writer
128	// prior to use.
129	func GetItemContentStream(id string, auth *Auth, output io.Writer) (uint64, error) {	22✔
130	// determine the size of the item	22✔
131	item, err := GetItem(id, auth)	22✔
132	if err != nil {	22✔
133	return 0, err	×
134	}	×
135
136	const downloadChunkSize = 10 * 1024 * 1024	22✔
137	downloadURL := fmt.Sprintf("/me/drive/items/%s/content", id)	22✔
138	if item.Size <= downloadChunkSize {	42✔
139	// simple one-shot download	20✔
140	content, err := Get(downloadURL, auth)	20✔
141	if err != nil {	20✔
142	return 0, err	×
143	}	×
144	n, err := output.Write(content)	20✔
145	return uint64(n), err	20✔
146	}
147
148	// multipart download
149	var n uint64	2✔
150	for i := 0; i < int(item.Size/downloadChunkSize)+1; i++ {	8✔
151	start := i * downloadChunkSize	6✔
152	end := start + downloadChunkSize - 1	6✔
153	log.Info().	6✔
154	Str("id", item.ID).	6✔
155	Str("name", item.Name).	6✔
156	Msgf("Downloading bytes %d-%d/%d.", start, end, item.Size)	6✔
157	content, err := Get(downloadURL, auth, Header{	6✔
158	key: "Range",	6✔
159	value: fmt.Sprintf("bytes=%d-%d", start, end),	6✔
160	})	6✔
161	if err != nil {	6✔
162	return n, err	×
163	}	×
164	written, err := output.Write(content)	6✔
165	n += uint64(written)	6✔
166	if err != nil {	6✔
167	return n, err	×
168	}	×
169	}
170	log.Info().	2✔
171	Str("id", item.ID).	2✔
172	Str("name", item.Name).	2✔
173	Uint64("size", n).	2✔
174	Msgf("Download completed!")	2✔
175	return n, nil	2✔
176	}
177
178	// Remove removes a directory or file by ID
179	func Remove(id string, auth *Auth) error {	12✔
180	return Delete("/me/drive/items/"+id, auth)	12✔
181	}	12✔
182
183	// Mkdir creates a directory on the server at the specified parent ID.
184	func Mkdir(name string, parentID string, auth Auth) (DriveItem, error) {	28✔
185	// create a new folder on the server	28✔
186	newFolderPost := DriveItem{	28✔
187	Name: name,	28✔
188	Folder: &Folder{},	28✔
189	}	28✔
190	bytePayload, _ := json.Marshal(newFolderPost)	28✔
191	resp, err := Post(childrenPathID(parentID), auth, bytes.NewReader(bytePayload))	28✔
192	if err != nil {	30✔
193	return nil, err	2✔
194	}	2✔
195	err = json.Unmarshal(resp, &newFolderPost)	26✔
196	return &newFolderPost, err	26✔
197	}
198
199	// Rename moves and/or renames an item on the server. The itemName and parentID
200	// arguments correspond to the new basename or id of the parent.
201	func Rename(itemID string, itemName string, parentID string, auth *Auth) error {	18✔
202	// start creating patch content for server	18✔
203	// mutex does not need to be initialized since it is never used locally	18✔
204	patchContent := DriveItem{	18✔
205	ConflictBehavior: "replace", // overwrite existing content at new location	18✔
206	Name: itemName,	18✔
207	Parent: &DriveItemParent{	18✔
208	ID: parentID,	18✔
209	},	18✔
210	}	18✔
211		18✔
212	// apply patch to server copy - note that we don't actually care about the	18✔
213	// response content, only if it returns an error	18✔
214	jsonPatch, _ := json.Marshal(patchContent)	18✔
215	_, err := Patch("/me/drive/items/"+itemID, auth, bytes.NewReader(jsonPatch))	18✔
216	if err != nil && strings.Contains(err.Error(), "resourceModified") {	18✔
217	// Wait a second, then retry the request. The Onedrive servers sometimes	×
218	// aren't quick enough here if the object has been recently created	×
219	// (<1 second ago).	×
220	time.Sleep(time.Second)	×
221	_, err = Patch("/me/drive/items/"+itemID, auth, bytes.NewReader(jsonPatch))	×
222	}	×
223	return err	18✔
224	}
225
226	// only used for parsing
227	type driveChildren struct {
228	Children []*DriveItem `json:"value"`
229	NextLink string `json:"@odata.nextLink"`
230	}
231
232	// this is the internal method that actually fetches an item's children
233	func getItemChildren(pollURL string, auth Auth) ([]DriveItem, error) {	45✔
234	fetched := make([]*DriveItem, 0)	45✔
235	for pollURL != "" {	96✔
236	body, err := Get(pollURL, auth)	51✔
237	if err != nil {	51✔
238	return fetched, err	×
239	}	×
240	var pollResult driveChildren	51✔
241	json.Unmarshal(body, &pollResult)	51✔
242		51✔
243	// there can be multiple pages of 200 items each (default).	51✔
244	// continue to next interation if we have an @odata.nextLink value	51✔
245	fetched = append(fetched, pollResult.Children...)	51✔
246	pollURL = strings.TrimPrefix(pollResult.NextLink, GraphURL)	51✔
247	}
248	return fetched, nil	45✔
249	}
250
251	// GetItemChildren fetches all children of an item denoted by ID.
252	func GetItemChildren(id string, auth Auth) ([]DriveItem, error) {	41✔
253	return getItemChildren(childrenPathID(id), auth)	41✔
254	}	41✔
255
256	// GetItemChildrenPath fetches all children of an item denoted by path.
257	func GetItemChildrenPath(path string, auth Auth) ([]DriveItem, error) {	4✔
258	return getItemChildren(childrenPath(path), auth)	4✔
259	}	4✔

jstaf / onedriver / 6554536784

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous