15665742047

Committed 15 Jun 2025 05:34PM UTC coverage: 76.321% (+5.6%) from 70.76%

Build # 15665742047

Build Type

push

github

Committed by

web-flow

Commit Message

Refactor the API and add WriteFrom (#2)

* feat: update dependencies and improve S3 uploader functionality

- Added `golang.org/x/sync v0.15.0` as an indirect dependency.
- Refactored tests in `server_test.go` to use the new `Write` method instead of `Put`.
- Removed unused multipart upload tests to streamline the test suite.
- Updated `Prefix` struct to eliminate unnecessary context handling.
- Refactored `Uploader` to improve error handling and context management during uploads.
- Introduced `calculatePartSize` function to optimize part size based on total size.
- Enhanced `UploadFrom` method to support context cancellation and improved concurrency handling.

* chore: reorganize require statements in go.mod for clarity

* feat: update dependencies and improve S3 uploader functionality

- Added `golang.org/x/sync v0.15.0` as an indirect dependency.
- Refactored tests in `server_test.go` to use the new `Write` method instead of `Put`.
- Removed unused multipart upload tests to streamline the test suite.
- Updated `Prefix` struct to eliminate unnecessary context handling.
- Refactored `Uploader` to improve error handling and context management during uploads.
- Introduced `calculatePartSize` function to optimize part size based on total size.
- Enhanced `UploadFrom` method to support context cancellation and improved concurrency handling.

* chore: reorganize require statements in go.mod for clarity

* fix: correct bucket initialization and method call in README; add uploader tests

* refactor: streamline error handling in uploader tests for clarity

* refactor: enhance code readability by improving comments and structuring in Bucket and Prefix implementations

* refactor: consolidate and enhance mock S3 server tests; remove redundant tests and improve utility function coverage

Run Details

111 of 131 new or added lines in 3 files covered. (84.73%)

83 existing lines in 5 files now uncovered.

2369 of 3104 relevant lines covered (76.32%)

486.36 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

79.15

/reader.go

// Copyright 2023 Sneller, Inc.
// Copyright 2025 Roman Atachiants
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

// Package s3 implements a lightweight
// client of the AWS S3 API.
//
// The Reader type can be used to view
// S3 objects as an io.Reader or io.ReaderAt.
package s3

import (
        "context"
        "errors"
        "fmt"
        "io"
        "io/fs"
        "net"
        "net/http"
        "net/url"
        "strings"
        "time"

        "github.com/kelindar/s3/aws"
)

// DefaultClient is the default HTTP client
// used for requests made from this package.
var DefaultClient = http.Client{
        Transport: &http.Transport{
                ResponseHeaderTimeout: 60 * time.Second,
                // Empirically, AWS creates about 40
                // DNS entries for S3, so 5 connections
                // per host is about 100 total connections.
                // (Note that the default here is 2!)
                MaxIdleConnsPerHost: 5,
                // Don't set Accept-Encoding: gzip
                // because it leads to the go client natively
                // decompressing gzipped objects.
                DisableCompression: true,
                // AWS S3 occasionally provides "dead" hosts
                // in their round-robin DNS responses, and the
                // fastest way to identify them is during
                // connection establishment:
                DialContext: (&net.Dialer{
                        Timeout: 2 * time.Second,
                }).DialContext,
        },
}

var (
        // ErrInvalidBucket is returned from calls that attempt
        // to use a bucket name that isn't valid according to
        // the S3 specification.
        ErrInvalidBucket = errors.New("invalid bucket name")
        // ErrETagChanged is returned from read operations where
        // the ETag of the underlying file has changed since
        // the file handle was constructed. (This package guarantees
        // that file read operations are always consistent with respect
        // to the ETag originally associated with the file handle.)
        ErrETagChanged = errors.New("file ETag changed")
)

func badBucket(name string) error {
        return fmt.Errorf("%w: %s", ErrInvalidBucket, name)
}

// ValidBucket returns whether or not
// bucket is a valid bucket name.
//
// See https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html
//
// Note: ValidBucket does not allow '.' characters,
// since bucket names containing dots are not accessible
// over HTTPS. (AWS docs say "not recommended for uses other than static website hosting.")
func ValidBucket(bucket string) bool {
        if len(bucket) < 3 || len(bucket) > 63 {
                return false
        }
        if strings.HasPrefix(bucket, "xn--") {
                return false
        }
        if strings.HasSuffix(bucket, "-s3alias") {
                return false
        }
        for i := 0; i < len(bucket); i++ {
                if bucket[i] >= 'a' && bucket[i] <= 'z' {
                        continue
                }
                if bucket[i] >= '0' && bucket[i] <= '9' {
                        continue
                }
                if i > 0 && i < len(bucket)-1 {
                        if bucket[i] == '-' {
                                continue
                        }
                        if bucket[i] == '.' && bucket[i-1] != '.' {
                                continue
                        }
                }
                return false
        }
        return true
}

// Reader presents a read-only view of an S3 object
type Reader struct {
        // Key is the sigining key that
        // Reader uses to make HTTP requests.
        // The key may have to be refreshed
        // every so often (see aws.SigningKey.Expired)
        Key *aws.SigningKey `xml:"-"`

        // Client is the HTTP client used to
        // make HTTP requests. By default it is
        // populated with DefaultClient, but
        // it may be set to any reasonable http client
        // implementation.
        Client *http.Client `xml:"-"`

        // ETag is the ETag of the object in S3
        // as returned by listing or a HEAD operation.
        ETag string `xml:"ETag"`
        // LastModified is the object's LastModified time
        // as returned by listing or a HEAD operation.
        LastModified time.Time `xml:"LastModified"`
        // Size is the object size in bytes.
        // It is populated on Open.
        Size int64 `xml:"Size"`
        // Bucket is the S3 bucket holding the object.
        Bucket string `xml:"-"`
        // Path is the S3 object key.
        Path string `xml:"Key"`
}

// rawURI produces a URI with a pre-escaped path+query string
func rawURI(k *aws.SigningKey, bucket string, query string) string {
        endPoint := k.BaseURI
        if endPoint == "" {
                // use virtual-host style if the bucket is compatible
                // (fallback to path-style if not)
                if strings.IndexByte(bucket, '.') < 0 {
                        return "https://" + bucket + ".s3." + k.Region + ".amazonaws.com" + "/" + query
                } else {
                        return "https://s3." + k.Region + ".amazonaws.com" + "/" + bucket + "/" + query
                }
        }
        return endPoint + "/" + bucket + "/" + query
}

// perform S3-specific path escaping;
// all the special characters are turned
// into their quoted bits, but we turn %2F
// back into / because AWS accepts those
// as part of the URI
func almostPathEscape(s string) string {
        return strings.ReplaceAll(queryEscape(s), "%2F", "/")
}

func queryEscape(s string) string {
        return strings.ReplaceAll(url.QueryEscape(s), "+", "%20")
}

// uri produces a URI by path-escaping the object string
// and passing it to rawURI (see also almostPathEscape)
func uri(k *aws.SigningKey, bucket, object string) string {
        return rawURI(k, bucket, almostPathEscape(object))
}

// URL returns a signed URL for a bucket and object
// that can be used directly with http.Get.
func URL(k *aws.SigningKey, bucket, object string) (string, error) {
        if !ValidBucket(bucket) {
                return "", badBucket(bucket)
        }
        return k.SignURL(uri(k, bucket, object), 1*time.Hour)
}

// Stat performs a HEAD on an S3 object
// and returns an associated Reader.
func Stat(k *aws.SigningKey, bucket, object string) (*Reader, error) {
        r := new(Reader)
        body, err := r.open(k, bucket, object, false)
        if body != nil {
                body.Close()
        }
        return r, err
}

// NewFile constructs a File that points to the given
// bucket, object, etag, and file size. The caller is
// assumed to have correctly determined these attributes
// in advance; this call does not perform any I/O to verify
// that the provided object exists or has a matching ETag
// and size.
func NewFile(k *aws.SigningKey, bucket, object, etag string, size int64) *File {
        return &File{
                Reader: Reader{
                        Key:    k,
                        Bucket: bucket,
                        Path:   object,
                        ETag:   etag,
                        Size:   size,
                },
                ctx: context.Background(),
        }
}

// Open performs a GET on an S3 object
// and returns the associated File.
func Open(k *aws.SigningKey, bucket, object string, contents bool) (*File, error) {
        f := new(File)
        err := f.open(k, bucket, object, contents)
        if err != nil {
                return nil, err
        }
        return f, nil
}

func flakyDo(cl *http.Client, req *http.Request) (*http.Response, error) {
        hasBody := req.Body != nil
        if cl == nil {
                cl = &DefaultClient
        }
        res, err := cl.Do(req)
        if err == nil && (res.StatusCode != 500 && res.StatusCode != 503) {
                return res, err
        }
        if hasBody && req.GetBody == nil {
                // can't re-do this request because
                // we can't rewind the Body reader
                return res, err
        }
        if res != nil {
                res.Body.Close()
        }
        if hasBody {
                req.Body, err = req.GetBody()
                if err != nil {
                        return nil, fmt.Errorf("req.GetBody: %w", err)
                }
        }
        return cl.Do(req)
}

func (f *File) open(k *aws.SigningKey, bucket, object string, contents bool) error {
        body, err := f.Reader.open(k, bucket, object, true)
        if err != nil {
                if body != nil {
                        body.Close()
                }
                return err
        }
        if !contents {
                body.Close()
                body = nil
        }
        f.body = body
        f.ctx = context.Background()
        return nil
}

func (r *Reader) open(k *aws.SigningKey, bucket, object string, contents bool) (io.ReadCloser, error) {
        if !ValidBucket(bucket) {
                return nil, badBucket(bucket)
        }
        method := http.MethodHead
        if contents {
                method = http.MethodGet
        }
        req, err := http.NewRequest(method, uri(k, bucket, object), nil)
        if err != nil {
                return nil, err
        }
        k.SignV4(req, nil)

        // FIXME: configurable http.Client here?
        res, err := flakyDo(&DefaultClient, req)
        if err != nil {
                return nil, err
        }
        if res.StatusCode != 200 {
                var inner error
                switch res.StatusCode {
                case 404:
                        inner = fs.ErrNotExist
                case 403:
                        inner = fs.ErrPermission
                default:
                        // NOTE: we can't extractMessage() here, because HEAD
                        // errors do not produce a response with an error message
                        inner = fmt.Errorf("s3.Open: %s returned %s", req.Method, res.Status)
                }
                err := &fs.PathError{
                        Op:   "open",
                        Path: "s3://" + bucket + "/" + object,
                        Err:  inner,
                }
                return res.Body, err
        }
        if res.ContentLength < 0 {
                return res.Body, fmt.Errorf("s3.Open: content length %d invalid", res.ContentLength)
        }
        lm, _ := time.Parse(time.RFC1123, res.Header.Get("LastModified"))
        *r = Reader{
                Key:          k,
                Client:       &DefaultClient,
                ETag:         res.Header.Get("ETag"),
                LastModified: lm,
                Size:         res.ContentLength,
                Bucket:       bucket,
                Path:         object,
        }
        return res.Body, nil
}

// WriteTo implements io.WriterTo
func (r *Reader) WriteTo(w io.Writer) (int64, error) {
        req, err := http.NewRequest("GET", uri(r.Key, r.Bucket, r.Path), nil)
        if err != nil {
                return 0, err
        }
        r.Key.SignV4(req, nil)

        res, err := flakyDo(r.Client, req)
        if err != nil {
                return 0, err
        }
        defer res.Body.Close()
        if res.StatusCode != 200 {
                return 0, fmt.Errorf("s3.Reader.WriteTo: status %s %q", res.Status, extractMessage(res.Body))
        }
        return io.Copy(w, res.Body)
}

// RangeReader produces an io.ReadCloser that reads
// bytes in the range from [off, off+width)
//
// It is the caller's responsibility to call Close()
// on the returned io.ReadCloser.
func (r *Reader) RangeReader(off, width int64) (io.ReadCloser, error) {
        req, err := http.NewRequest("GET", uri(r.Key, r.Bucket, r.Path), nil)
        if err != nil {
                return nil, err
        }
        req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", off, off+width-1))
        req.Header.Set("If-Match", r.ETag)
        r.Key.SignV4(req, nil)
        res, err := flakyDo(r.Client, req)
        if err != nil {
                return nil, err
        }
        switch res.StatusCode {
        default:
                defer res.Body.Close()
                return nil, fmt.Errorf("s3.Reader.RangeReader: status %s %q", res.Status, extractMessage(res.Body))
        case http.StatusPreconditionFailed:
                res.Body.Close()
                return nil, ErrETagChanged
        case http.StatusNotFound:
                res.Body.Close()
                return nil, &fs.PathError{Op: "read", Path: r.Path, Err: fs.ErrNotExist}
        case http.StatusPartialContent, http.StatusOK:
                // okay; fallthrough
        }
        return res.Body, nil
}

// ReadAt implements io.ReaderAt
func (r *Reader) ReadAt(dst []byte, off int64) (int, error) {
        rd, err := r.RangeReader(off, int64(len(dst)))
        if err != nil {
                return 0, err
        }
        defer rd.Close()
        return io.ReadFull(rd, dst)
}

// BucketRegion returns the region associated
// with the given bucket.
func BucketRegion(k *aws.SigningKey, bucket string) (string, error) {
        if !ValidBucket(bucket) {
                return "", badBucket(bucket)
        }
        if k.BaseURI != "" {
                return k.Region, nil
        }
        uri := rawURI(k, bucket, "")
        req, err := http.NewRequest(http.MethodHead, uri, nil)
        if err != nil {
                return "", err
        }
        k.SignV4(req, nil)
        res, err := flakyDo(&DefaultClient, req)
        if err != nil {
                return "", err
        }
        defer res.Body.Close()
        if res.StatusCode == 403 {
                return k.Region, nil
        }
        if res.StatusCode != 200 && res.StatusCode != 301 {
                return "", fmt.Errorf("s3.BucketRegion: %s %q", res.Status, extractMessage(res.Body))
        }
        region := res.Header.Get("x-amz-bucket-region")
        if region == "" {
                return k.Region, nil
        }
        return region, nil
}

// DeriveForBucket can be passed to aws.AmbientCreds
// as a DeriveFn that automatically re-derives keys
// so that they apply to the region in which the
// given bucket lives.
func DeriveForBucket(bucket string) aws.DeriveFn {
        return func(baseURI, id, secret, token, region, service string) (*aws.SigningKey, error) {
                if !ValidBucket(bucket) {
                        return nil, badBucket(bucket)
                }

                if service != "s3" && service != "b2" {
                        return nil, fmt.Errorf("s3.DeriveForBucket: expected service \"s3\"; got %q", service)
                }

                k := aws.DeriveKey(baseURI, id, secret, region, service)
                k.Token = token
                bregion, err := BucketRegion(k, bucket)
                if err != nil {
                        return nil, err
                }
                if bregion == region {
                        return k, nil
                }
                k = aws.DeriveKey(baseURI, id, secret, bregion, service)
                k.Token = token
                return k, nil
        }
}

1	// Copyright 2023 Sneller, Inc.
2	// Copyright 2025 Roman Atachiants
3	//
4	// Licensed under the Apache License, Version 2.0 (the "License");
5	// you may not use this file except in compliance with the License.
6	// You may obtain a copy of the License at
7	//
8	// http://www.apache.org/licenses/LICENSE-2.0
9	//
10	// Unless required by applicable law or agreed to in writing, software
11	// distributed under the License is distributed on an "AS IS" BASIS,
12	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	// See the License for the specific language governing permissions and
14	// limitations under the License.
15
16	// Package s3 implements a lightweight
17	// client of the AWS S3 API.
18	//
19	// The Reader type can be used to view
20	// S3 objects as an io.Reader or io.ReaderAt.
21	package s3
22
23	import (
24	"context"
25	"errors"
26	"fmt"
27	"io"
28	"io/fs"
29	"net"
30	"net/http"
31	"net/url"
32	"strings"
33	"time"
34
35	"github.com/kelindar/s3/aws"
36	)
37
38	// DefaultClient is the default HTTP client
39	// used for requests made from this package.
40	var DefaultClient = http.Client{
41	Transport: &http.Transport{
42	ResponseHeaderTimeout: 60 * time.Second,
43	// Empirically, AWS creates about 40
44	// DNS entries for S3, so 5 connections
45	// per host is about 100 total connections.
46	// (Note that the default here is 2!)
47	MaxIdleConnsPerHost: 5,
48	// Don't set Accept-Encoding: gzip
49	// because it leads to the go client natively
50	// decompressing gzipped objects.
51	DisableCompression: true,
52	// AWS S3 occasionally provides "dead" hosts
53	// in their round-robin DNS responses, and the
54	// fastest way to identify them is during
55	// connection establishment:
56	DialContext: (&net.Dialer{
57	Timeout: 2 * time.Second,
58	}).DialContext,
59	},
60	}
61
62	var (
63	// ErrInvalidBucket is returned from calls that attempt
64	// to use a bucket name that isn't valid according to
65	// the S3 specification.
66	ErrInvalidBucket = errors.New("invalid bucket name")
67	// ErrETagChanged is returned from read operations where
68	// the ETag of the underlying file has changed since
69	// the file handle was constructed. (This package guarantees
70	// that file read operations are always consistent with respect
71	// to the ETag originally associated with the file handle.)
72	ErrETagChanged = errors.New("file ETag changed")
73	)
74
75	func badBucket(name string) error {	5✔
76	return fmt.Errorf("%w: %s", ErrInvalidBucket, name)	5✔
77	}	5✔
78
79	// ValidBucket returns whether or not
80	// bucket is a valid bucket name.
81	//
82	// See https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html
83	//
84	// Note: ValidBucket does not allow '.' characters,
85	// since bucket names containing dots are not accessible
86	// over HTTPS. (AWS docs say "not recommended for uses other than static website hosting.")
87	func ValidBucket(bucket string) bool {	162✔
88	if len(bucket) < 3 \|\| len(bucket) > 63 {	165✔
89	return false	3✔
90	}	3✔
91	if strings.HasPrefix(bucket, "xn--") {	160✔
92	return false	1✔
93	}	1✔
94	if strings.HasSuffix(bucket, "-s3alias") {	159✔
95	return false	1✔
96	}	1✔
97	for i := 0; i < len(bucket); i++ {	1,947✔
98	if bucket[i] >= 'a' && bucket[i] <= 'z' {	3,343✔
99	continue	1,553✔
100	}
101	if bucket[i] >= '0' && bucket[i] <= '9' {	309✔
102	continue	72✔
103	}
104	if i > 0 && i < len(bucket)-1 {	326✔
105	if bucket[i] == '-' {	304✔
106	continue	143✔
107	}
108	if bucket[i] == '.' && bucket[i-1] != '.' {	29✔
109	continue	11✔
110	}
111	}
112	return false	11✔
113	}
114	return true	146✔
115	}
116
117	// Reader presents a read-only view of an S3 object
118	type Reader struct {
119	// Key is the sigining key that
120	// Reader uses to make HTTP requests.
121	// The key may have to be refreshed
122	// every so often (see aws.SigningKey.Expired)
123	Key *aws.SigningKey `xml:"-"`
124
125	// Client is the HTTP client used to
126	// make HTTP requests. By default it is
127	// populated with DefaultClient, but
128	// it may be set to any reasonable http client
129	// implementation.
130	Client *http.Client `xml:"-"`
131
132	// ETag is the ETag of the object in S3
133	// as returned by listing or a HEAD operation.
134	ETag string `xml:"ETag"`
135	// LastModified is the object's LastModified time
136	// as returned by listing or a HEAD operation.
137	LastModified time.Time `xml:"LastModified"`
138	// Size is the object size in bytes.
139	// It is populated on Open.
140	Size int64 `xml:"Size"`
141	// Bucket is the S3 bucket holding the object.
142	Bucket string `xml:"-"`
143	// Path is the S3 object key.
144	Path string `xml:"Key"`
145	}
146
147	// rawURI produces a URI with a pre-escaped path+query string
148	func rawURI(k *aws.SigningKey, bucket string, query string) string {	148✔
149	endPoint := k.BaseURI	148✔
150	if endPoint == "" {	150✔
151	// use virtual-host style if the bucket is compatible	2✔
152	// (fallback to path-style if not)	2✔
153	if strings.IndexByte(bucket, '.') < 0 {	4✔
154	return "https://" + bucket + ".s3." + k.Region + ".amazonaws.com" + "/" + query	2✔
155	} else {	2✔
156	return "https://s3." + k.Region + ".amazonaws.com" + "/" + bucket + "/" + query	×
157	}	×
158	}
159	return endPoint + "/" + bucket + "/" + query	146✔
160	}
161
162	// perform S3-specific path escaping;
163	// all the special characters are turned
164	// into their quoted bits, but we turn %2F
165	// back into / because AWS accepts those
166	// as part of the URI
167	func almostPathEscape(s string) string {	129✔
168	return strings.ReplaceAll(queryEscape(s), "%2F", "/")	129✔
169	}	129✔
170
171	func queryEscape(s string) string {	206✔
172	return strings.ReplaceAll(url.QueryEscape(s), "+", "%20")	206✔
173	}	206✔
174
175	// uri produces a URI by path-escaping the object string
176	// and passing it to rawURI (see also almostPathEscape)
177	func uri(k *aws.SigningKey, bucket, object string) string {	77✔
178	return rawURI(k, bucket, almostPathEscape(object))	77✔
179	}	77✔
180
181	// URL returns a signed URL for a bucket and object
182	// that can be used directly with http.Get.
183	func URL(k *aws.SigningKey, bucket, object string) (string, error) {	2✔
184	if !ValidBucket(bucket) {	3✔
185	return "", badBucket(bucket)	1✔
186	}	1✔
187	return k.SignURL(uri(k, bucket, object), 1*time.Hour)	1✔
188	}
189
190	// Stat performs a HEAD on an S3 object
191	// and returns an associated Reader.
192	func Stat(k aws.SigningKey, bucket, object string) (Reader, error) {	1✔
193	r := new(Reader)	1✔
194	body, err := r.open(k, bucket, object, false)	1✔
195	if body != nil {	2✔
196	body.Close()	1✔
197	}	1✔
198	return r, err	1✔
199	}
200
201	// NewFile constructs a File that points to the given
202	// bucket, object, etag, and file size. The caller is
203	// assumed to have correctly determined these attributes
204	// in advance; this call does not perform any I/O to verify
205	// that the provided object exists or has a matching ETag
206	// and size.
207	func NewFile(k aws.SigningKey, bucket, object, etag string, size int64) File {	1✔
208	return &File{	1✔
209	Reader: Reader{	1✔
210	Key: k,	1✔
211	Bucket: bucket,	1✔
212	Path: object,	1✔
213	ETag: etag,	1✔
214	Size: size,	1✔
215	},	1✔
216	ctx: context.Background(),	1✔
217	}	1✔
218	}	1✔
219
220	// Open performs a GET on an S3 object
221	// and returns the associated File.
222	func Open(k aws.SigningKey, bucket, object string, contents bool) (File, error) {	22✔
223	f := new(File)	22✔
224	err := f.open(k, bucket, object, contents)	22✔
225	if err != nil {	32✔
226	return nil, err	10✔
227	}	10✔
228	return f, nil	12✔
229	}
230
231	func flakyDo(cl http.Client, req http.Request) (*http.Response, error) {	182✔
232	hasBody := req.Body != nil	182✔
233	if cl == nil {	183✔
234	cl = &DefaultClient	1✔
235	}	1✔
236	res, err := cl.Do(req)	182✔
237	if err == nil && (res.StatusCode != 500 && res.StatusCode != 503) {	362✔
238	return res, err	180✔
239	}	180✔
240	if hasBody && req.GetBody == nil {	2✔
241	// can't re-do this request because	×
242	// we can't rewind the Body reader	×
243	return res, err	×
244	}	×
245	if res != nil {	2✔
UNCOV 246	res.Body.Close()	×
UNCOV 247	}	×
248	if hasBody {	3✔
249	req.Body, err = req.GetBody()	1✔
250	if err != nil {	1✔
251	return nil, fmt.Errorf("req.GetBody: %w", err)	×
252	}	×
253	}
254	return cl.Do(req)	2✔
255	}
256
257	func (f File) open(k aws.SigningKey, bucket, object string, contents bool) error {	22✔
258	body, err := f.Reader.open(k, bucket, object, true)	22✔
259	if err != nil {	32✔
260	if body != nil {	20✔
261	body.Close()	10✔
262	}	10✔
263	return err	10✔
264	}
265	if !contents {	22✔
266	body.Close()	10✔
267	body = nil	10✔
268	}	10✔
269	f.body = body	12✔
270	f.ctx = context.Background()	12✔
271	return nil	12✔
272	}
273
274	func (r Reader) open(k aws.SigningKey, bucket, object string, contents bool) (io.ReadCloser, error) {	23✔
275	if !ValidBucket(bucket) {	23✔
276	return nil, badBucket(bucket)	×
277	}	×
278	method := http.MethodHead	23✔
279	if contents {	45✔
280	method = http.MethodGet	22✔
281	}	22✔
282	req, err := http.NewRequest(method, uri(k, bucket, object), nil)	23✔
283	if err != nil {	23✔
284	return nil, err	×
285	}	×
286	k.SignV4(req, nil)	23✔
287		23✔
288	// FIXME: configurable http.Client here?	23✔
289	res, err := flakyDo(&DefaultClient, req)	23✔
290	if err != nil {	23✔
291	return nil, err	×
292	}	×
293	if res.StatusCode != 200 {	33✔
294	var inner error	10✔
295	switch res.StatusCode {	10✔
296	case 404:	10✔
297	inner = fs.ErrNotExist	10✔
298	case 403:	×
299	inner = fs.ErrPermission	×
300	default:	×
301	// NOTE: we can't extractMessage() here, because HEAD	×
302	// errors do not produce a response with an error message	×
303	inner = fmt.Errorf("s3.Open: %s returned %s", req.Method, res.Status)	×
304	}
305	err := &fs.PathError{	10✔
306	Op: "open",	10✔
307	Path: "s3://" + bucket + "/" + object,	10✔
308	Err: inner,	10✔
309	}	10✔
310	return res.Body, err	10✔
311	}
312	if res.ContentLength < 0 {	13✔
313	return res.Body, fmt.Errorf("s3.Open: content length %d invalid", res.ContentLength)	×
314	}	×
315	lm, _ := time.Parse(time.RFC1123, res.Header.Get("LastModified"))	13✔
316	*r = Reader{	13✔
317	Key: k,	13✔
318	Client: &DefaultClient,	13✔
319	ETag: res.Header.Get("ETag"),	13✔
320	LastModified: lm,	13✔
321	Size: res.ContentLength,	13✔
322	Bucket: bucket,	13✔
323	Path: object,	13✔
324	}	13✔
325	return res.Body, nil	13✔
326	}
327
328	// WriteTo implements io.WriterTo
329	func (r *Reader) WriteTo(w io.Writer) (int64, error) {	3✔
330	req, err := http.NewRequest("GET", uri(r.Key, r.Bucket, r.Path), nil)	3✔
331	if err != nil {	3✔
332	return 0, err	×
333	}	×
334	r.Key.SignV4(req, nil)	3✔
335		3✔
336	res, err := flakyDo(r.Client, req)	3✔
337	if err != nil {	3✔
338	return 0, err	×
339	}	×
340	defer res.Body.Close()	3✔
341	if res.StatusCode != 200 {	5✔
342	return 0, fmt.Errorf("s3.Reader.WriteTo: status %s %q", res.Status, extractMessage(res.Body))	2✔
343	}	2✔
344	return io.Copy(w, res.Body)	1✔
345	}
346
347	// RangeReader produces an io.ReadCloser that reads
348	// bytes in the range from [off, off+width)
349	//
350	// It is the caller's responsibility to call Close()
351	// on the returned io.ReadCloser.
352	func (r *Reader) RangeReader(off, width int64) (io.ReadCloser, error) {	16✔
353	req, err := http.NewRequest("GET", uri(r.Key, r.Bucket, r.Path), nil)	16✔
354	if err != nil {	16✔
355	return nil, err	×
356	}	×
357	req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", off, off+width-1))	16✔
358	req.Header.Set("If-Match", r.ETag)	16✔
359	r.Key.SignV4(req, nil)	16✔
360	res, err := flakyDo(r.Client, req)	16✔
361	if err != nil {	16✔
362	return nil, err	×
363	}	×
364	switch res.StatusCode {	16✔
365	default:	1✔
366	defer res.Body.Close()	1✔
367	return nil, fmt.Errorf("s3.Reader.RangeReader: status %s %q", res.Status, extractMessage(res.Body))	1✔
368	case http.StatusPreconditionFailed:	×
369	res.Body.Close()	×
370	return nil, ErrETagChanged	×
371	case http.StatusNotFound:	3✔
372	res.Body.Close()	3✔
373	return nil, &fs.PathError{Op: "read", Path: r.Path, Err: fs.ErrNotExist}	3✔
374	case http.StatusPartialContent, http.StatusOK:	12✔
375	// okay; fallthrough
376	}
377	return res.Body, nil	12✔
378	}
379
380	// ReadAt implements io.ReaderAt
381	func (r *Reader) ReadAt(dst []byte, off int64) (int, error) {	2✔
382	rd, err := r.RangeReader(off, int64(len(dst)))	2✔
383	if err != nil {	3✔
384	return 0, err	1✔
385	}	1✔
386	defer rd.Close()	1✔
387	return io.ReadFull(rd, dst)	1✔
388	}
389
390	// BucketRegion returns the region associated
391	// with the given bucket.
392	func BucketRegion(k *aws.SigningKey, bucket string) (string, error) {	6✔
393	if !ValidBucket(bucket) {	8✔
394	return "", badBucket(bucket)	2✔
395	}	2✔
396	if k.BaseURI != "" {	7✔
397	return k.Region, nil	3✔
398	}	3✔
399	uri := rawURI(k, bucket, "")	1✔
400	req, err := http.NewRequest(http.MethodHead, uri, nil)	1✔
401	if err != nil {	1✔
402	return "", err	×
403	}	×
404	k.SignV4(req, nil)	1✔
405	res, err := flakyDo(&DefaultClient, req)	1✔
406	if err != nil {	1✔
407	return "", err	×
408	}	×
409	defer res.Body.Close()	1✔
410	if res.StatusCode == 403 {	2✔
411	return k.Region, nil	1✔
412	}	1✔
413	if res.StatusCode != 200 && res.StatusCode != 301 {	×
414	return "", fmt.Errorf("s3.BucketRegion: %s %q", res.Status, extractMessage(res.Body))	×
415	}	×
416	region := res.Header.Get("x-amz-bucket-region")	×
417	if region == "" {	×
418	return k.Region, nil	×
419	}	×
420	return region, nil	×
421	}
422
423	// DeriveForBucket can be passed to aws.AmbientCreds
424	// as a DeriveFn that automatically re-derives keys
425	// so that they apply to the region in which the
426	// given bucket lives.
427	func DeriveForBucket(bucket string) aws.DeriveFn {	1✔
428	return func(baseURI, id, secret, token, region, service string) (*aws.SigningKey, error) {	4✔
429	if !ValidBucket(bucket) {	3✔
430	return nil, badBucket(bucket)	×
431	}	×
432
433	if service != "s3" && service != "b2" {	5✔
434	return nil, fmt.Errorf("s3.DeriveForBucket: expected service \"s3\"; got %q", service)	2✔
435	}	2✔
436
437	k := aws.DeriveKey(baseURI, id, secret, region, service)	1✔
438	k.Token = token	1✔
439	bregion, err := BucketRegion(k, bucket)	1✔
440	if err != nil {	1✔
441	return nil, err	×
442	}	×
443	if bregion == region {	2✔
444	return k, nil	1✔
445	}	1✔
446	k = aws.DeriveKey(baseURI, id, secret, bregion, service)	×
447	k.Token = token	×
448	return k, nil	×
449	}
450	}

kelindar / s3 / 15665742047

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous