12587214381

Committed 02 Jan 2025 06:46PM UTC coverage: 49.501% (+0.2%) from 49.314%

Build # 12587214381

Build Type

push

github

Committed by

bmflynn

Commit Message

Merge branch '8-checksum-failure-when-downloading-using-edl-token'

Run Details

11 of 14 new or added lines in 1 file covered. (78.57%)

101 existing lines in 9 files now uncovered.

794 of 1604 relevant lines covered (49.5%)

0.57 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

73.5

/internal/fetch_http.go

package internal

import (
        "bufio"
        "context"
        "errors"
        "fmt"
        "io"
        "net/http"
        "net/http/cookiejar"
        "os"
        "sync"
        "time"

        "github.com/jdxcode/netrc"
)

var defaultNetrcFinder = findNetrc

type FailedDownload struct {
        RequestID    string
        ResponseBody string
        Status       string
        URL          string
}

func newFailedDownloadError(resp *http.Response) *FailedDownload {
        var body string
        dat, err := io.ReadAll(resp.Body)
        if err == nil {
                body = string(dat)
        }
        url := ""
        if resp.Request != nil {
                url = resp.Request.URL.String()
        }
        return &FailedDownload{
                RequestID:    resp.Header.Get("request-id"),
                ResponseBody: body,
                Status:       resp.Status,
                URL:          url,
        }
}

func (e *FailedDownload) Error() string {
        rid := e.RequestID
        if rid == "" {
                rid = "<unavailable>"
        }
        return fmt.Sprintf("%s requestid=%s", e.Status, rid)
}

func ResolveEDLToken(token string) string {
        // Check for token; commandline flag has priority over env var
        resolvedToken := token
        if resolvedToken == "" {
                // Check env var if commandline flag not set
                bearer, ok := os.LookupEnv("EDL_TOKEN")
                if ok && bearer != "" {
                        resolvedToken = bearer
                }
        }
        return resolvedToken
}

// Sets basic auth on redirect if the host is in the netrc file.
func newRedirectWithNetrcCredentials() (func(*http.Request, []*http.Request) error, error) {
        fpath, err := defaultNetrcFinder()
        if err != nil {
                return nil, err
        }
        nc, err := netrc.Parse(fpath)
        if err != nil {
                return nil, fmt.Errorf("failed to read netrc: %w", err)
        }
        mu := &sync.Mutex{}
        return func(req *http.Request, via []*http.Request) error {
                host := req.URL.Hostname()
                mu.Lock()
                machine := nc.Machine(host)
                mu.Unlock()
                if machine != nil {
                        req.SetBasicAuth(machine.Get("login"), machine.Get("password"))
                }
                return nil
        }, nil
}

// HTTPFetch is a Fetcher that supports basic file fetching. It supports netrc for authentication
// redirects and uses an in-memory cookie jar to save authentication cookies provided by
// authentication services such as NASA Einternal.
type HTTPFetcher struct {
        client   *http.Client
        readSize int64
        // If provided an authorization header is added to every request
        bearerToken string
}

func NewHTTPFetcher(netrc bool, edlToken string) (*HTTPFetcher, error) {
        client := &http.Client{
                Timeout: 20 * time.Minute,
        }

        // Token has priority over netrc if set
        if edlToken == "" && netrc {
                // Netrc needs a cookiejar so we don't have to do redirect everytime
                jar, err := cookiejar.New(nil)
                if err != nil {
                        return nil, fmt.Errorf("creating cookiejar: %w", err)
                }
                client.Jar = jar
                client.CheckRedirect, err = newRedirectWithNetrcCredentials()
                if err != nil {
                        return nil, fmt.Errorf("configuring netrc token redirect: %w", err)
                }
        }
        return &HTTPFetcher{
                client:      client,
                readSize:    2 << 19,
                bearerToken: edlToken,
        }, nil
}

func (f *HTTPFetcher) newRequest(ctx context.Context, url string) (*http.Request, error) {
        req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
        if err != nil {
                return nil, err
        }
        if f.bearerToken != "" {
                if req.URL.Scheme != "https" {
                        return nil, fmt.Errorf("refusing to add bearer token to non-https url %s", req.URL)
                }
                req.Header.Add("Authorization", "Bearer "+f.bearerToken)
        }
        return req, nil
}

// Fetch url to destdir using url's basename as the filename and update hash with the file
// bytes as they are read.
func (f *HTTPFetcher) Fetch(ctx context.Context, url string, w io.Writer) (int64, error) {
        req, err := f.newRequest(ctx, url)
        if err != nil {
                return 0, err
        }

        resp, err := f.client.Do(req)
        if err != nil {
                return 0, err
        }
        if resp.StatusCode != http.StatusOK {
                err = newFailedDownloadError(resp)
                resp.Body.Close()
                return 0, err
        }
        defer resp.Body.Close()

        var size int64
        buf := make([]byte, f.readSize)
        r := bufio.NewReader(resp.Body)
        for {
                n, rErr := r.Read(buf)
                _, wErr := w.Write(buf[:n])
                if wErr != nil {
                        return size, fmt.Errorf("writing to file: %w", err)
                }

                size += int64(n)

                if errors.Is(rErr, io.EOF) {
                        break
                }
                if rErr != nil {
                        return size, fmt.Errorf("reading from remote: %w", rErr)
                }
        }
        return size, nil
}

1	package internal
2
3	import (
4	"bufio"
5	"context"
6	"errors"
7	"fmt"
8	"io"
9	"net/http"
10	"net/http/cookiejar"
11	"os"
12	"sync"
13	"time"
14
15	"github.com/jdxcode/netrc"
16	)
17
18	var defaultNetrcFinder = findNetrc
19
20	type FailedDownload struct {
21	RequestID string
22	ResponseBody string
23	Status string
24	URL string
25	}
26
27	func newFailedDownloadError(resp http.Response) FailedDownload {	1✔
28	var body string	1✔
29	dat, err := io.ReadAll(resp.Body)	1✔
30	if err == nil {	2✔
31	body = string(dat)	1✔
32	}	1✔
33	url := ""	1✔
34	if resp.Request != nil {	2✔
35	url = resp.Request.URL.String()	1✔
36	}	1✔
37	return &FailedDownload{	1✔
38	RequestID: resp.Header.Get("request-id"),	1✔
39	ResponseBody: body,	1✔
40	Status: resp.Status,	1✔
41	URL: url,	1✔
42	}	1✔
43	}
44
45	func (e *FailedDownload) Error() string {	1✔
46	rid := e.RequestID	1✔
47	if rid == "" {	1✔
48	rid = "<unavailable>"	×
49	}	×
50	return fmt.Sprintf("%s requestid=%s", e.Status, rid)	1✔
51	}
52
53	func ResolveEDLToken(token string) string {	×
54	// Check for token; commandline flag has priority over env var	×
55	resolvedToken := token	×
56	if resolvedToken == "" {	×
57	// Check env var if commandline flag not set	×
58	bearer, ok := os.LookupEnv("EDL_TOKEN")	×
59	if ok && bearer != "" {	×
60	resolvedToken = bearer	×
61	}	×
62	}
63	return resolvedToken	×
64	}
65
66	// Sets basic auth on redirect if the host is in the netrc file.
67	func newRedirectWithNetrcCredentials() (func(http.Request, []http.Request) error, error) {	1✔
68	fpath, err := defaultNetrcFinder()	1✔
69	if err != nil {	1✔
70	return nil, err	×
71	}	×
72	nc, err := netrc.Parse(fpath)	1✔
73	if err != nil {	1✔
74	return nil, fmt.Errorf("failed to read netrc: %w", err)	×
75	}	×
76	mu := &sync.Mutex{}	1✔
77	return func(req http.Request, via []http.Request) error {	2✔
78	host := req.URL.Hostname()	1✔
79	mu.Lock()	1✔
80	machine := nc.Machine(host)	1✔
81	mu.Unlock()	1✔
82	if machine != nil {	2✔
83	req.SetBasicAuth(machine.Get("login"), machine.Get("password"))	1✔
84	}	1✔
85	return nil	1✔
86	}, nil
87	}
88
89	// HTTPFetch is a Fetcher that supports basic file fetching. It supports netrc for authentication
90	// redirects and uses an in-memory cookie jar to save authentication cookies provided by
91	// authentication services such as NASA Einternal.
92	type HTTPFetcher struct {
93	client *http.Client
94	readSize int64
95	// If provided an authorization header is added to every request
96	bearerToken string
97	}
98
99	func NewHTTPFetcher(netrc bool, edlToken string) (*HTTPFetcher, error) {	1✔
100	client := &http.Client{	1✔
101	Timeout: 20 * time.Minute,	1✔
102	}	1✔
103		1✔
104	// Token has priority over netrc if set	1✔
105	if edlToken == "" && netrc {	2✔
106	// Netrc needs a cookiejar so we don't have to do redirect everytime	1✔
107	jar, err := cookiejar.New(nil)	1✔
108	if err != nil {	1✔
109	return nil, fmt.Errorf("creating cookiejar: %w", err)	×
110	}	×
111	client.Jar = jar	1✔
112	client.CheckRedirect, err = newRedirectWithNetrcCredentials()	1✔
113	if err != nil {	1✔
NEW 114	return nil, fmt.Errorf("configuring netrc token redirect: %w", err)	×
115	}	×
116	}
117	return &HTTPFetcher{	1✔
118	client: client,	1✔
119	readSize: 2 << 19,	1✔
120	bearerToken: edlToken,	1✔
121	}, nil	1✔
122	}
123
124	func (f HTTPFetcher) newRequest(ctx context.Context, url string) (http.Request, error) {	1✔
125	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)	1✔
126	if err != nil {	1✔
NEW 127	return nil, err	×
NEW 128	}	×
129	if f.bearerToken != "" {	2✔
130	if req.URL.Scheme != "https" {	2✔
131	return nil, fmt.Errorf("refusing to add bearer token to non-https url %s", req.URL)	1✔
132	}	1✔
133	req.Header.Add("Authorization", "Bearer "+f.bearerToken)	1✔
134	}
135	return req, nil	1✔
136	}
137
138	// Fetch url to destdir using url's basename as the filename and update hash with the file
139	// bytes as they are read.
140	func (f *HTTPFetcher) Fetch(ctx context.Context, url string, w io.Writer) (int64, error) {	1✔
141	req, err := f.newRequest(ctx, url)	1✔
142	if err != nil {	1✔
143	return 0, err	×
144	}	×
145
146	resp, err := f.client.Do(req)	1✔
147	if err != nil {	1✔
148	return 0, err	×
149	}	×
150	if resp.StatusCode != http.StatusOK {	2✔
151	err = newFailedDownloadError(resp)	1✔
152	resp.Body.Close()	1✔
153	return 0, err	1✔
154	}	1✔
155	defer resp.Body.Close()	1✔
156		1✔
157	var size int64	1✔
158	buf := make([]byte, f.readSize)	1✔
159	r := bufio.NewReader(resp.Body)	1✔
160	for {	2✔
161	n, rErr := r.Read(buf)	1✔
162	_, wErr := w.Write(buf[:n])	1✔
163	if wErr != nil {	1✔
164	return size, fmt.Errorf("writing to file: %w", err)	×
165	}	×
166
167	size += int64(n)	1✔
168		1✔
169	if errors.Is(rErr, io.EOF) {	2✔
170	break	1✔
171	}
172	if rErr != nil {	×
173	return size, fmt.Errorf("reading from remote: %w", rErr)	×
174	}	×
175	}
176	return size, nil	1✔
177	}

bmflynn / cmrfetch / 12587214381

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous