10115894847

Committed 26 Jul 2024 06:24PM UTC coverage: 86.092% (+0.1%) from 85.961%

Build # 10115894847

Build Type

push

github

Committed by

bmflynn

Commit Message

remove unnecessary location validation

Run Details

3 of 3 new or added lines in 1 file covered. (100.0%)

1 existing line in 1 file now uncovered.

749 of 870 relevant lines covered (86.09%)

0.97 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

82.69

/internal/fetch_http.go

package internal

import (
        "bufio"
        "context"
        "errors"
        "fmt"
        "io"
        "io/ioutil"
        "net/http"
        "net/http/cookiejar"
        "sync"
        "time"

        "github.com/jdxcode/netrc"
)

var defaultNetrcFinder = findNetrc

type FailedDownload struct {
        RequestID    string
        ResponseBody string
        Status       string
        URL          string
}

func newFailedDownloadError(resp *http.Response) *FailedDownload {
        var body string
        dat, err := ioutil.ReadAll(resp.Body)
        if err == nil {
                body = string(dat)
        }
        url := ""
        if resp.Request != nil {
                url = resp.Request.URL.String()
        }
        return &FailedDownload{
                RequestID:    resp.Header.Get("request-id"),
                ResponseBody: body,
                Status:       resp.Status,
                URL:          url,
        }
}

func (e *FailedDownload) Error() string {
        rid := e.RequestID
        if rid == "" {
                rid = "<unavailable>"
        }
        return fmt.Sprintf("%s requestid=%s", e.Status, rid)
}

// Sets basic auth on redirect if the host is in the netrc file.
func newRedirectWithNetrcCredentials() (func(*http.Request, []*http.Request) error, error) {
        fpath, err := defaultNetrcFinder()
        if err != nil {
                return nil, err
        }
        nc, err := netrc.Parse(fpath)
        if err != nil {
                return nil, fmt.Errorf("failed to read netrc: %w", err)
        }
        mu := &sync.Mutex{}
        return func(req *http.Request, via []*http.Request) error {
                host := req.URL.Hostname()
                mu.Lock()
                machine := nc.Machine(host)
                mu.Unlock()
                if machine != nil {
                        req.SetBasicAuth(machine.Get("login"), machine.Get("password"))
                }
                return nil
        }, nil
}

// HTTPFetch is a Fetcher that supports basic file fetching. It supports netrc for authentication
// redirects and uses an in-memory cookie jar to save authentication cookies provided by
// authentication services such as NASA Einternal.
type HTTPFetcher struct {
        client   *http.Client
        readSize int64
}

func NewHTTPFetcher(netrc bool) (*HTTPFetcher, error) {
        client := &http.Client{
                Timeout: 20 * time.Minute,
        }
        if netrc {
                // Netrc needs a cookiejar so we don't have to do redirect everytime
                jar, err := cookiejar.New(nil)
                if err != nil {
                        return nil, fmt.Errorf("creating cookiejar: %w", err)
                }
                client.Jar = jar
                client.CheckRedirect, err = newRedirectWithNetrcCredentials()
                if err != nil {
                        return nil, err
                }
        }
        return &HTTPFetcher{
                client:   client,
                readSize: 2 << 19,
        }, nil
}

func (f *HTTPFetcher) newRequest(ctx context.Context, url string) (*http.Request, error) {
        return http.NewRequestWithContext(ctx, "GET", url, nil)
}

// Fetch url to destdir using url's basename as the filename and update hash with the file
// bytes as they are read.
func (f *HTTPFetcher) Fetch(ctx context.Context, url string, w io.Writer) (int64, error) {
        req, err := f.newRequest(ctx, url)
        if err != nil {
                return 0, err
        }

        resp, err := f.client.Do(req)
        if resp.StatusCode != http.StatusOK {
                err = newFailedDownloadError(resp)
                resp.Body.Close()
                return 0, err
        }
        defer resp.Body.Close()

        // if err := validateResponse(req, resp); err != nil {
        //         return 0, err
        // }

        var size int64
        buf := make([]byte, f.readSize)
        r := bufio.NewReader(resp.Body)
        for {
                n, rErr := r.Read(buf)
                _, wErr := w.Write(buf[:n])
                if wErr != nil {
                        return size, fmt.Errorf("writing to file: %w", err)
                }

                size += int64(n)

                if errors.Is(rErr, io.EOF) {
                        break
                }
                if rErr != nil {
                        return size, fmt.Errorf("reading from remote: %w", rErr)
                }
        }
        return size, nil
}

func validateResponse(req *http.Request, resp *http.Response) error {
        wanted := req.URL.Hostname()
        found := resp.Request.URL.Hostname()
        if wanted != found {
                return fmt.Errorf("probable auth redirect error; expected host %s, found %s", wanted, found)
        }
        return nil
}

1	package internal
2
3	import (
4	"bufio"
5	"context"
6	"errors"
7	"fmt"
8	"io"
9	"io/ioutil"
10	"net/http"
11	"net/http/cookiejar"
12	"sync"
13	"time"
14
15	"github.com/jdxcode/netrc"
16	)
17
18	var defaultNetrcFinder = findNetrc
19
20	type FailedDownload struct {
21	RequestID string
22	ResponseBody string
23	Status string
24	URL string
25	}
26
27	func newFailedDownloadError(resp http.Response) FailedDownload {	1✔
28	var body string	1✔
29	dat, err := ioutil.ReadAll(resp.Body)	1✔
30	if err == nil {	2✔
31	body = string(dat)	1✔
32	}	1✔
33	url := ""	1✔
34	if resp.Request != nil {	2✔
35	url = resp.Request.URL.String()	1✔
36	}	1✔
37	return &FailedDownload{	1✔
38	RequestID: resp.Header.Get("request-id"),	1✔
39	ResponseBody: body,	1✔
40	Status: resp.Status,	1✔
41	URL: url,	1✔
42	}	1✔
43	}
44
45	func (e *FailedDownload) Error() string {	1✔
46	rid := e.RequestID	1✔
47	if rid == "" {	1✔
48	rid = "<unavailable>"	×
49	}	×
50	return fmt.Sprintf("%s requestid=%s", e.Status, rid)	1✔
51	}
52
53	// Sets basic auth on redirect if the host is in the netrc file.
54	func newRedirectWithNetrcCredentials() (func(http.Request, []http.Request) error, error) {	1✔
55	fpath, err := defaultNetrcFinder()	1✔
56	if err != nil {	1✔
57	return nil, err	×
58	}	×
59	nc, err := netrc.Parse(fpath)	1✔
60	if err != nil {	1✔
61	return nil, fmt.Errorf("failed to read netrc: %w", err)	×
62	}	×
63	mu := &sync.Mutex{}	1✔
64	return func(req http.Request, via []http.Request) error {	2✔
65	host := req.URL.Hostname()	1✔
66	mu.Lock()	1✔
67	machine := nc.Machine(host)	1✔
68	mu.Unlock()	1✔
69	if machine != nil {	2✔
70	req.SetBasicAuth(machine.Get("login"), machine.Get("password"))	1✔
71	}	1✔
72	return nil	1✔
73	}, nil
74	}
75
76	// HTTPFetch is a Fetcher that supports basic file fetching. It supports netrc for authentication
77	// redirects and uses an in-memory cookie jar to save authentication cookies provided by
78	// authentication services such as NASA Einternal.
79	type HTTPFetcher struct {
80	client *http.Client
81	readSize int64
82	}
83
84	func NewHTTPFetcher(netrc bool) (*HTTPFetcher, error) {	1✔
85	client := &http.Client{	1✔
86	Timeout: 20 * time.Minute,	1✔
87	}	1✔
88	if netrc {	2✔
89	// Netrc needs a cookiejar so we don't have to do redirect everytime	1✔
90	jar, err := cookiejar.New(nil)	1✔
91	if err != nil {	1✔
92	return nil, fmt.Errorf("creating cookiejar: %w", err)	×
93	}	×
94	client.Jar = jar	1✔
95	client.CheckRedirect, err = newRedirectWithNetrcCredentials()	1✔
96	if err != nil {	1✔
97	return nil, err	×
98	}	×
99	}
100	return &HTTPFetcher{	1✔
101	client: client,	1✔
102	readSize: 2 << 19,	1✔
103	}, nil	1✔
104	}
105
106	func (f HTTPFetcher) newRequest(ctx context.Context, url string) (http.Request, error) {	1✔
107	return http.NewRequestWithContext(ctx, "GET", url, nil)	1✔
108	}	1✔
109
110	// Fetch url to destdir using url's basename as the filename and update hash with the file
111	// bytes as they are read.
112	func (f *HTTPFetcher) Fetch(ctx context.Context, url string, w io.Writer) (int64, error) {	1✔
113	req, err := f.newRequest(ctx, url)	1✔
114	if err != nil {	1✔
115	return 0, err	×
116	}	×
117
118	resp, err := f.client.Do(req)	1✔
119	if resp.StatusCode != http.StatusOK {	2✔
120	err = newFailedDownloadError(resp)	1✔
121	resp.Body.Close()	1✔
122	return 0, err	1✔
123	}	1✔
124	defer resp.Body.Close()	1✔
125		1✔
126	// if err := validateResponse(req, resp); err != nil {	1✔
127	// return 0, err	1✔
128	// }	1✔
129		1✔
130	var size int64	1✔
131	buf := make([]byte, f.readSize)	1✔
132	r := bufio.NewReader(resp.Body)	1✔
133	for {	2✔
134	n, rErr := r.Read(buf)	1✔
135	_, wErr := w.Write(buf[:n])	1✔
136	if wErr != nil {	1✔
137	return size, fmt.Errorf("writing to file: %w", err)	×
138	}	×
139
140	size += int64(n)	1✔
141		1✔
142	if errors.Is(rErr, io.EOF) {	2✔
143	break	1✔
144	}
145	if rErr != nil {	×
146	return size, fmt.Errorf("reading from remote: %w", rErr)	×
147	}	×
148	}
149	return size, nil	1✔
150	}
151
152	func validateResponse(req http.Request, resp http.Response) error {	1✔
153	wanted := req.URL.Hostname()	1✔
154	found := resp.Request.URL.Hostname()	1✔
155	if wanted != found {	2✔
156	return fmt.Errorf("probable auth redirect error; expected host %s, found %s", wanted, found)	1✔
157	}	1✔
UNCOV 158	return nil	×
159	}

bmflynn / cmrfetch / 10115894847

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous