11181316829

Committed 04 Oct 2024 01:54PM UTC coverage: 84.305% (-0.9%) from 85.235%

Build # 11181316829

Build Type

push

github

Committed by

bmflynn

Commit Message

log auth types for verbose

Run Details

3 of 4 new or added lines in 2 files covered. (75.0%)

20 existing lines in 1 file now uncovered.

752 of 892 relevant lines covered (84.3%)

0.95 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

70.87

/internal/fetch_http.go

package internal

import (
        "bufio"
        "context"
        "errors"
        "fmt"
        "io"
        "net/http"
        "net/http/cookiejar"
        "os"
        "sync"
        "time"

        "github.com/jdxcode/netrc"
)

var defaultNetrcFinder = findNetrc
var edlToken = ""

type FailedDownload struct {
        RequestID    string
        ResponseBody string
        Status       string
        URL          string
}

func newFailedDownloadError(resp *http.Response) *FailedDownload {
        var body string
        dat, err := io.ReadAll(resp.Body)
        if err == nil {
                body = string(dat)
        }
        url := ""
        if resp.Request != nil {
                url = resp.Request.URL.String()
        }
        return &FailedDownload{
                RequestID:    resp.Header.Get("request-id"),
                ResponseBody: body,
                Status:       resp.Status,
                URL:          url,
        }
}

func (e *FailedDownload) Error() string {
        rid := e.RequestID
        if rid == "" {
                rid = "<unavailable>"
        }
        return fmt.Sprintf("%s requestid=%s", e.Status, rid)
}

func ResolveEDLToken(token string) string {
        // Check for token; commandline flag has priority over env var
        resolvedToken := token
        if resolvedToken == "" {
                // Check env var if commandline flag not set
                bearer, ok := os.LookupEnv("EDL_TOKEN")
                if ok && bearer != "" {
                        resolvedToken = bearer
                }
        }
        return resolvedToken
}

// Sets token auth header
func newRedirectWithToken(bearer string) (func(*http.Request, []*http.Request) error, error) {
        return func(req *http.Request, via []*http.Request) error {
                req.Header.Add("Authorization", "Bearer "+bearer)
                return nil
        }, nil
}

// Sets basic auth on redirect if the host is in the netrc file.
func newRedirectWithNetrcCredentials() (func(*http.Request, []*http.Request) error, error) {
        fpath, err := defaultNetrcFinder()
        if err != nil {
                return nil, err
        }
        nc, err := netrc.Parse(fpath)
        if err != nil {
                return nil, fmt.Errorf("failed to read netrc: %w", err)
        }
        mu := &sync.Mutex{}
        return func(req *http.Request, via []*http.Request) error {
                host := req.URL.Hostname()
                mu.Lock()
                machine := nc.Machine(host)
                mu.Unlock()
                if machine != nil {
                        req.SetBasicAuth(machine.Get("login"), machine.Get("password"))
                }
                return nil
        }, nil
}

// HTTPFetch is a Fetcher that supports basic file fetching. It supports netrc for authentication
// redirects and uses an in-memory cookie jar to save authentication cookies provided by
// authentication services such as NASA Einternal.
type HTTPFetcher struct {
        client   *http.Client
        readSize int64
}

func NewHTTPFetcher(netrc bool, edlToken string) (*HTTPFetcher, error) {
        client := &http.Client{
                Timeout: 20 * time.Minute,
        }

        // Token has priority over netrc if set
        if edlToken == "" && netrc {
                // Netrc needs a cookiejar so we don't have to do redirect everytime
                jar, err := cookiejar.New(nil)
                if err != nil {
                        return nil, fmt.Errorf("creating cookiejar: %w", err)
                }
                client.Jar = jar
                client.CheckRedirect, err = newRedirectWithNetrcCredentials()
                if err != nil {
                        return nil, err
                }
        }
        return &HTTPFetcher{
                client:   client,
                readSize: 2 << 19,
        }, nil
}

func (f *HTTPFetcher) newRequest(ctx context.Context, url string) (*http.Request, error) {
        return http.NewRequestWithContext(ctx, "GET", url, nil)
}

// Fetch url to destdir using url's basename as the filename and update hash with the file
// bytes as they are read.
func (f *HTTPFetcher) Fetch(ctx context.Context, url string, w io.Writer) (int64, error) {
        req, err := f.newRequest(ctx, url)
        if err != nil {
                return 0, err
        }

        if edlToken != "" {
                req.Header.Add("Authorization", "Bearer "+edlToken)
        }

        resp, err := f.client.Do(req)
        if err != nil {
                return 0, err
        }
        if resp.StatusCode != http.StatusOK {
                err = newFailedDownloadError(resp)
                resp.Body.Close()
                return 0, err
        }
        defer resp.Body.Close()

        // if err := validateResponse(req, resp); err != nil {
        //         return 0, err
        // }

        var size int64
        buf := make([]byte, f.readSize)
        r := bufio.NewReader(resp.Body)
        for {
                n, rErr := r.Read(buf)
                _, wErr := w.Write(buf[:n])
                if wErr != nil {
                        return size, fmt.Errorf("writing to file: %w", err)
                }

                size += int64(n)

                if errors.Is(rErr, io.EOF) {
                        break
                }
                if rErr != nil {
                        return size, fmt.Errorf("reading from remote: %w", rErr)
                }
        }
        return size, nil
}

func validateResponse(req *http.Request, resp *http.Response) error {
        wanted := req.URL.Hostname()
        found := resp.Request.URL.Hostname()
        if wanted != found {
                return fmt.Errorf("probable auth redirect error; expected host %s, found %s", wanted, found)
        }
        return nil
}

1	package internal
2
3	import (
4	"bufio"
5	"context"
6	"errors"
7	"fmt"
8	"io"
9	"net/http"
10	"net/http/cookiejar"
11	"os"
12	"sync"
13	"time"
14
15	"github.com/jdxcode/netrc"
16	)
17
18	var defaultNetrcFinder = findNetrc
19	var edlToken = ""
20
21	type FailedDownload struct {
22	RequestID string
23	ResponseBody string
24	Status string
25	URL string
26	}
27
28	func newFailedDownloadError(resp http.Response) FailedDownload {	1✔
29	var body string	1✔
30	dat, err := io.ReadAll(resp.Body)	1✔
31	if err == nil {	2✔
32	body = string(dat)	1✔
33	}	1✔
34	url := ""	1✔
35	if resp.Request != nil {	2✔
36	url = resp.Request.URL.String()	1✔
37	}	1✔
38	return &FailedDownload{	1✔
39	RequestID: resp.Header.Get("request-id"),	1✔
40	ResponseBody: body,	1✔
41	Status: resp.Status,	1✔
42	URL: url,	1✔
43	}	1✔
44	}
45
46	func (e *FailedDownload) Error() string {	1✔
47	rid := e.RequestID	1✔
48	if rid == "" {	1✔
UNCOV 49	rid = "<unavailable>"	×
50	}	×
51	return fmt.Sprintf("%s requestid=%s", e.Status, rid)	1✔
52	}
53
NEW 54	func ResolveEDLToken(token string) string {	×
UNCOV 55	// Check for token; commandline flag has priority over env var	×
UNCOV 56	resolvedToken := token	×
UNCOV 57	if resolvedToken == "" {	×
UNCOV 58	// Check env var if commandline flag not set	×
UNCOV 59	bearer, ok := os.LookupEnv("EDL_TOKEN")	×
UNCOV 60	if ok && bearer != "" {	×
UNCOV 61	resolvedToken = bearer	×
62	}	×
63	}
UNCOV 64	return resolvedToken	×
65	}
66
67	// Sets token auth header
UNCOV 68	func newRedirectWithToken(bearer string) (func(http.Request, []http.Request) error, error) {	×
69	return func(req http.Request, via []http.Request) error {	×
70	req.Header.Add("Authorization", "Bearer "+bearer)	×
71	return nil	×
72	}, nil	×
73	}
74
75	// Sets basic auth on redirect if the host is in the netrc file.
76	func newRedirectWithNetrcCredentials() (func(http.Request, []http.Request) error, error) {	1✔
77	fpath, err := defaultNetrcFinder()	1✔
78	if err != nil {	1✔
UNCOV 79	return nil, err	×
80	}	×
81	nc, err := netrc.Parse(fpath)	1✔
82	if err != nil {	1✔
UNCOV 83	return nil, fmt.Errorf("failed to read netrc: %w", err)	×
84	}	×
85	mu := &sync.Mutex{}	1✔
86	return func(req http.Request, via []http.Request) error {	2✔
87	host := req.URL.Hostname()	1✔
88	mu.Lock()	1✔
89	machine := nc.Machine(host)	1✔
90	mu.Unlock()	1✔
91	if machine != nil {	2✔
92	req.SetBasicAuth(machine.Get("login"), machine.Get("password"))	1✔
93	}	1✔
94	return nil	1✔
95	}, nil
96	}
97
98	// HTTPFetch is a Fetcher that supports basic file fetching. It supports netrc for authentication
99	// redirects and uses an in-memory cookie jar to save authentication cookies provided by
100	// authentication services such as NASA Einternal.
101	type HTTPFetcher struct {
102	client *http.Client
103	readSize int64
104	}
105
106	func NewHTTPFetcher(netrc bool, edlToken string) (*HTTPFetcher, error) {	1✔
107	client := &http.Client{	1✔
108	Timeout: 20 * time.Minute,	1✔
109	}	1✔
110		1✔
111	// Token has priority over netrc if set	1✔
112	if edlToken == "" && netrc {	2✔
113	// Netrc needs a cookiejar so we don't have to do redirect everytime	1✔
114	jar, err := cookiejar.New(nil)	1✔
115	if err != nil {	1✔
UNCOV 116	return nil, fmt.Errorf("creating cookiejar: %w", err)	×
117	}	×
118	client.Jar = jar	1✔
119	client.CheckRedirect, err = newRedirectWithNetrcCredentials()	1✔
120	if err != nil {	1✔
UNCOV 121	return nil, err	×
122	}	×
123	}
124	return &HTTPFetcher{	1✔
125	client: client,	1✔
126	readSize: 2 << 19,	1✔
127	}, nil	1✔
128	}
129
130	func (f HTTPFetcher) newRequest(ctx context.Context, url string) (http.Request, error) {	1✔
131	return http.NewRequestWithContext(ctx, "GET", url, nil)	1✔
132	}	1✔
133
134	// Fetch url to destdir using url's basename as the filename and update hash with the file
135	// bytes as they are read.
136	func (f *HTTPFetcher) Fetch(ctx context.Context, url string, w io.Writer) (int64, error) {	1✔
137	req, err := f.newRequest(ctx, url)	1✔
138	if err != nil {	1✔
UNCOV 139	return 0, err	×
140	}	×
141
142	if edlToken != "" {	1✔
UNCOV 143	req.Header.Add("Authorization", "Bearer "+edlToken)	×
144	}	×
145
146	resp, err := f.client.Do(req)	1✔
147	if err != nil {	1✔
UNCOV 148	return 0, err	×
149	}	×
150	if resp.StatusCode != http.StatusOK {	2✔
151	err = newFailedDownloadError(resp)	1✔
152	resp.Body.Close()	1✔
153	return 0, err	1✔
154	}	1✔
155	defer resp.Body.Close()	1✔
156		1✔
157	// if err := validateResponse(req, resp); err != nil {	1✔
158	// return 0, err	1✔
159	// }	1✔
160		1✔
161	var size int64	1✔
162	buf := make([]byte, f.readSize)	1✔
163	r := bufio.NewReader(resp.Body)	1✔
164	for {	2✔
165	n, rErr := r.Read(buf)	1✔
166	_, wErr := w.Write(buf[:n])	1✔
167	if wErr != nil {	1✔
UNCOV 168	return size, fmt.Errorf("writing to file: %w", err)	×
169	}	×
170
171	size += int64(n)	1✔
172		1✔
173	if errors.Is(rErr, io.EOF) {	2✔
174	break	1✔
175	}
UNCOV 176	if rErr != nil {	×
177	return size, fmt.Errorf("reading from remote: %w", rErr)	×
178	}	×
179	}
180	return size, nil	1✔
181	}
182
183	func validateResponse(req http.Request, resp http.Response) error {	1✔
184	wanted := req.URL.Hostname()	1✔
185	found := resp.Request.URL.Hostname()	1✔
186	if wanted != found {	2✔
187	return fmt.Errorf("probable auth redirect error; expected host %s, found %s", wanted, found)	1✔
188	}	1✔
UNCOV 189	return nil	×
190	}

bmflynn / cmrfetch / 11181316829

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous