14371540395

Committed 10 Apr 2025 03:18AM UTC coverage: 70.031% (-2.2%) from 72.207%

Build # 14371540395

Build Type

push

github

Committed by

web-flow

Commit Message

Add file operations to all container types (#4)

* Add DeleteFile methods to containers and standardize file operations

- Add DeleteFile method to FTP container
- Add file operations (get, save, list, delete) to SSH container
- Add file operations (get, save, list, delete) to Localstack (S3) container
- Add comprehensive tests for all new methods
- Update README with examples for all file operations

* Increase test timeout to 300s for container tests

Run Details

164 of 256 new or added lines in 3 files covered. (64.06%)

107 existing lines in 3 files now uncovered.

680 of 971 relevant lines covered (70.03%)

3.09 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

80.13

/containers/localstack.go

package containers

import (
        "bytes"
        "context"
        "fmt"
        "io"
        "os"
        "path/filepath"
        "strings"
        "sync/atomic"
        "testing"
        "time"

        "github.com/aws/aws-sdk-go-v2/aws"
        "github.com/aws/aws-sdk-go-v2/config"
        "github.com/aws/aws-sdk-go-v2/credentials"
        "github.com/aws/aws-sdk-go-v2/service/s3"
        "github.com/aws/aws-sdk-go-v2/service/s3/types"
        "github.com/stretchr/testify/require"
        "github.com/testcontainers/testcontainers-go"
        "github.com/testcontainers/testcontainers-go/wait"
)

// LocalstackTestContainer is a wrapper around a testcontainers.Container that provides an S3 endpoint
type LocalstackTestContainer struct {
        Container testcontainers.Container
        Endpoint  string
        counter   atomic.Int64
}

// NewLocalstackTestContainer creates a new Localstack test container and returns a LocalstackTestContainer instance
func NewLocalstackTestContainer(ctx context.Context, t *testing.T) *LocalstackTestContainer {
        req := testcontainers.ContainerRequest{
                Image:        "localstack/localstack:3.0.0",
                ExposedPorts: []string{"4566/tcp"},
                Env: map[string]string{
                        "SERVICES":              "s3",
                        "DEFAULT_REGION":        "us-east-1",
                        "AWS_ACCESS_KEY_ID":     "test",
                        "AWS_SECRET_ACCESS_KEY": "test",
                },
                WaitingFor: wait.ForAll(
                        wait.ForListeningPort("4566/tcp"),
                        wait.ForLog("Ready."),
                ).WithDeadline(time.Minute),
        }

        container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
                ContainerRequest: req,
                Started:          true,
        })
        require.NoError(t, err)

        host, err := container.Host(ctx)
        require.NoError(t, err)

        port, err := container.MappedPort(ctx, "4566")
        require.NoError(t, err)

        endpoint := fmt.Sprintf("http://%s:%s", host, port.Port())
        return &LocalstackTestContainer{
                Container: container,
                Endpoint:  endpoint,
        }
}

// MakeS3Connection creates a new S3 connection using the test container endpoint and returns the connection and a bucket name
func (lc *LocalstackTestContainer) MakeS3Connection(ctx context.Context, t *testing.T) (client *s3.Client, bucketName string) {
        cfg, err := config.LoadDefaultConfig(ctx,
                config.WithRegion("us-east-1"),
                config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test", "test", "")),
        )
        require.NoError(t, err)

        client = s3.NewFromConfig(cfg, func(o *s3.Options) {
                o.BaseEndpoint = aws.String(lc.Endpoint)
                o.UsePathStyle = true
        })

        bucketName = fmt.Sprintf("test-bucket-%d-%d", time.Now().UnixNano(), lc.counter.Add(1))
        _, err = client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(bucketName)})
        require.NoError(t, err)

        return client, bucketName
}

// createS3Client creates an S3 client connected to the test container
func (lc *LocalstackTestContainer) createS3Client(ctx context.Context) (*s3.Client, error) {
        cfg, err := config.LoadDefaultConfig(ctx,
                config.WithRegion("us-east-1"),
                config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test", "test", "")),
        )
        if err != nil {
                return nil, fmt.Errorf("failed to load AWS config: %w", err)
        }

        client := s3.NewFromConfig(cfg, func(o *s3.Options) {
                o.BaseEndpoint = aws.String(lc.Endpoint)
                o.UsePathStyle = true
        })

        return client, nil
}

// GetFile downloads a file from S3
func (lc *LocalstackTestContainer) GetFile(ctx context.Context, bucketName, objectKey, localPath string) error {
        client, err := lc.createS3Client(ctx)
        if err != nil {
                return fmt.Errorf("failed to create S3 client: %w", err)
        }

        localDir := filepath.Dir(localPath)
        if err := os.MkdirAll(localDir, 0o750); err != nil {
                return fmt.Errorf("failed to create local directory %s: %w", localDir, err)
        }

        if !strings.HasPrefix(filepath.Clean(localPath), filepath.Clean(localDir)) {
                return fmt.Errorf("localPath %s attempts to escape from directory %s", localPath, localDir)
        }

        // get object from S3
        output, err := client.GetObject(ctx, &s3.GetObjectInput{
                Bucket: aws.String(bucketName),
                Key:    aws.String(objectKey),
        })
        if err != nil {
                return fmt.Errorf("failed to get object %s from bucket %s: %w", objectKey, bucketName, err)
        }
        defer output.Body.Close()

        // create local file
        file, err := os.OpenFile(localPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600)
        if err != nil {
                return fmt.Errorf("failed to create local file %s: %w", localPath, err)
        }
        defer file.Close()

        // copy object content to local file
        if _, err := io.Copy(file, output.Body); err != nil {
                return fmt.Errorf("failed to copy content from S3 object to local file: %w", err)
        }

        return nil
}

// SaveFile uploads a file to S3
func (lc *LocalstackTestContainer) SaveFile(ctx context.Context, localPath, bucketName, objectKey string) error {
        client, err := lc.createS3Client(ctx)
        if err != nil {
                return fmt.Errorf("failed to create S3 client: %w", err)
        }

        if !strings.HasPrefix(filepath.Clean(localPath), filepath.Clean(filepath.Dir(localPath))) {
                return fmt.Errorf("localPath %s attempts to escape from its directory", localPath)
        }

        // read local file
        fileData, err := os.ReadFile(localPath)
        if err != nil {
                return fmt.Errorf("failed to read local file %s: %w", localPath, err)
        }

        // upload to S3
        _, err = client.PutObject(ctx, &s3.PutObjectInput{
                Bucket: aws.String(bucketName),
                Key:    aws.String(objectKey),
                Body:   bytes.NewReader(fileData),
        })
        if err != nil {
                return fmt.Errorf("failed to upload file to S3: %w", err)
        }

        return nil
}

// ListFiles lists objects in an S3 bucket, optionally with a prefix
func (lc *LocalstackTestContainer) ListFiles(ctx context.Context, bucketName, prefix string) ([]types.Object, error) {
        client, err := lc.createS3Client(ctx)
        if err != nil {
                return nil, fmt.Errorf("failed to create S3 client: %w", err)
        }

        // list objects
        input := &s3.ListObjectsV2Input{
                Bucket: aws.String(bucketName),
        }

        // add prefix if provided
        if prefix != "" {
                input.Prefix = aws.String(prefix)
        }

        output, err := client.ListObjectsV2(ctx, input)
        if err != nil {
                return nil, fmt.Errorf("failed to list objects in bucket %s: %w", bucketName, err)
        }

        return output.Contents, nil
}

// DeleteFile deletes an object from S3
func (lc *LocalstackTestContainer) DeleteFile(ctx context.Context, bucketName, objectKey string) error {
        client, err := lc.createS3Client(ctx)
        if err != nil {
                return fmt.Errorf("failed to create S3 client: %w", err)
        }

        // delete object
        _, err = client.DeleteObject(ctx, &s3.DeleteObjectInput{
                Bucket: aws.String(bucketName),
                Key:    aws.String(objectKey),
        })
        if err != nil {
                return fmt.Errorf("failed to delete object %s from bucket %s: %w", objectKey, bucketName, err)
        }

        return nil
}

// Close terminates the container
func (lc *LocalstackTestContainer) Close(ctx context.Context) error {
        return lc.Container.Terminate(ctx)
}

1	package containers
2
3	import (
4	"bytes"
5	"context"
6	"fmt"
7	"io"
8	"os"
9	"path/filepath"
10	"strings"
11	"sync/atomic"
12	"testing"
13	"time"
14
15	"github.com/aws/aws-sdk-go-v2/aws"
16	"github.com/aws/aws-sdk-go-v2/config"
17	"github.com/aws/aws-sdk-go-v2/credentials"
18	"github.com/aws/aws-sdk-go-v2/service/s3"
19	"github.com/aws/aws-sdk-go-v2/service/s3/types"
20	"github.com/stretchr/testify/require"
21	"github.com/testcontainers/testcontainers-go"
22	"github.com/testcontainers/testcontainers-go/wait"
23	)
24
25	// LocalstackTestContainer is a wrapper around a testcontainers.Container that provides an S3 endpoint
26	type LocalstackTestContainer struct {
27	Container testcontainers.Container
28	Endpoint string
29	counter atomic.Int64
30	}
31
32	// NewLocalstackTestContainer creates a new Localstack test container and returns a LocalstackTestContainer instance
33	func NewLocalstackTestContainer(ctx context.Context, t testing.T) LocalstackTestContainer {	5✔
34	req := testcontainers.ContainerRequest{	5✔
35	Image: "localstack/localstack:3.0.0",	5✔
36	ExposedPorts: []string{"4566/tcp"},	5✔
37	Env: map[string]string{	5✔
38	"SERVICES": "s3",	5✔
39	"DEFAULT_REGION": "us-east-1",	5✔
40	"AWS_ACCESS_KEY_ID": "test",	5✔
41	"AWS_SECRET_ACCESS_KEY": "test",	5✔
42	},	5✔
43	WaitingFor: wait.ForAll(	5✔
44	wait.ForListeningPort("4566/tcp"),	5✔
45	wait.ForLog("Ready."),	5✔
46	).WithDeadline(time.Minute),	5✔
47	}	5✔
48		5✔
49	container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{	5✔
50	ContainerRequest: req,	5✔
51	Started: true,	5✔
52	})	5✔
53	require.NoError(t, err)	5✔
54		5✔
55	host, err := container.Host(ctx)	5✔
56	require.NoError(t, err)	5✔
57		5✔
58	port, err := container.MappedPort(ctx, "4566")	5✔
59	require.NoError(t, err)	5✔
60		5✔
61	endpoint := fmt.Sprintf("http://%s:%s", host, port.Port())	5✔
62	return &LocalstackTestContainer{	5✔
63	Container: container,	5✔
64	Endpoint: endpoint,	5✔
65	}	5✔
66	}	5✔
67
68	// MakeS3Connection creates a new S3 connection using the test container endpoint and returns the connection and a bucket name
69	func (lc LocalstackTestContainer) MakeS3Connection(ctx context.Context, t testing.T) (client *s3.Client, bucketName string) {	5✔
70	cfg, err := config.LoadDefaultConfig(ctx,	5✔
71	config.WithRegion("us-east-1"),	5✔
72	config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test", "test", "")),	5✔
73	)	5✔
74	require.NoError(t, err)	5✔
75		5✔
76	client = s3.NewFromConfig(cfg, func(o *s3.Options) {	10✔
77	o.BaseEndpoint = aws.String(lc.Endpoint)	5✔
78	o.UsePathStyle = true	5✔
79	})	5✔
80
81	bucketName = fmt.Sprintf("test-bucket-%d-%d", time.Now().UnixNano(), lc.counter.Add(1))	5✔
82	_, err = client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(bucketName)})	5✔
83	require.NoError(t, err)	5✔
84		5✔
85	return client, bucketName	5✔
86	}
87
88	// createS3Client creates an S3 client connected to the test container
89	func (lc LocalstackTestContainer) createS3Client(ctx context.Context) (s3.Client, error) {	12✔
90	cfg, err := config.LoadDefaultConfig(ctx,	12✔
91	config.WithRegion("us-east-1"),	12✔
92	config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test", "test", "")),	12✔
93	)	12✔
94	if err != nil {	12✔
NEW UNCOV 95	return nil, fmt.Errorf("failed to load AWS config: %w", err)	×
NEW UNCOV 96	}	×
97
98	client := s3.NewFromConfig(cfg, func(o *s3.Options) {	24✔
99	o.BaseEndpoint = aws.String(lc.Endpoint)	12✔
100	o.UsePathStyle = true	12✔
101	})	12✔
102
103	return client, nil	12✔
104	}
105
106	// GetFile downloads a file from S3
107	func (lc *LocalstackTestContainer) GetFile(ctx context.Context, bucketName, objectKey, localPath string) error {	1✔
108	client, err := lc.createS3Client(ctx)	1✔
109	if err != nil {	1✔
NEW UNCOV 110	return fmt.Errorf("failed to create S3 client: %w", err)	×
NEW UNCOV 111	}	×
112
113	localDir := filepath.Dir(localPath)	1✔
114	if err := os.MkdirAll(localDir, 0o750); err != nil {	1✔
NEW UNCOV 115	return fmt.Errorf("failed to create local directory %s: %w", localDir, err)	×
NEW UNCOV 116	}	×
117
118	if !strings.HasPrefix(filepath.Clean(localPath), filepath.Clean(localDir)) {	1✔
NEW UNCOV 119	return fmt.Errorf("localPath %s attempts to escape from directory %s", localPath, localDir)	×
NEW UNCOV 120	}	×
121
122	// get object from S3
123	output, err := client.GetObject(ctx, &s3.GetObjectInput{	1✔
124	Bucket: aws.String(bucketName),	1✔
125	Key: aws.String(objectKey),	1✔
126	})	1✔
127	if err != nil {	1✔
NEW UNCOV 128	return fmt.Errorf("failed to get object %s from bucket %s: %w", objectKey, bucketName, err)	×
NEW UNCOV 129	}	×
130	defer output.Body.Close()	1✔
131		1✔
132	// create local file	1✔
133	file, err := os.OpenFile(localPath, os.O_CREATE\|os.O_WRONLY\|os.O_TRUNC, 0o600)	1✔
134	if err != nil {	1✔
NEW UNCOV 135	return fmt.Errorf("failed to create local file %s: %w", localPath, err)	×
NEW UNCOV 136	}	×
137	defer file.Close()	1✔
138		1✔
139	// copy object content to local file	1✔
140	if _, err := io.Copy(file, output.Body); err != nil {	1✔
NEW UNCOV 141	return fmt.Errorf("failed to copy content from S3 object to local file: %w", err)	×
NEW UNCOV 142	}	×
143
144	return nil	1✔
145	}
146
147	// SaveFile uploads a file to S3
148	func (lc *LocalstackTestContainer) SaveFile(ctx context.Context, localPath, bucketName, objectKey string) error {	4✔
149	client, err := lc.createS3Client(ctx)	4✔
150	if err != nil {	4✔
NEW UNCOV 151	return fmt.Errorf("failed to create S3 client: %w", err)	×
NEW UNCOV 152	}	×
153
154	if !strings.HasPrefix(filepath.Clean(localPath), filepath.Clean(filepath.Dir(localPath))) {	4✔
NEW UNCOV 155	return fmt.Errorf("localPath %s attempts to escape from its directory", localPath)	×
NEW UNCOV 156	}	×
157
158	// read local file
159	fileData, err := os.ReadFile(localPath)	4✔
160	if err != nil {	4✔
NEW UNCOV 161	return fmt.Errorf("failed to read local file %s: %w", localPath, err)	×
NEW UNCOV 162	}	×
163
164	// upload to S3
165	_, err = client.PutObject(ctx, &s3.PutObjectInput{	4✔
166	Bucket: aws.String(bucketName),	4✔
167	Key: aws.String(objectKey),	4✔
168	Body: bytes.NewReader(fileData),	4✔
169	})	4✔
170	if err != nil {	4✔
NEW UNCOV 171	return fmt.Errorf("failed to upload file to S3: %w", err)	×
NEW UNCOV 172	}	×
173
174	return nil	4✔
175	}
176
177	// ListFiles lists objects in an S3 bucket, optionally with a prefix
178	func (lc *LocalstackTestContainer) ListFiles(ctx context.Context, bucketName, prefix string) ([]types.Object, error) {	5✔
179	client, err := lc.createS3Client(ctx)	5✔
180	if err != nil {	5✔
NEW UNCOV 181	return nil, fmt.Errorf("failed to create S3 client: %w", err)	×
NEW UNCOV 182	}	×
183
184	// list objects
185	input := &s3.ListObjectsV2Input{	5✔
186	Bucket: aws.String(bucketName),	5✔
187	}	5✔
188		5✔
189	// add prefix if provided	5✔
190	if prefix != "" {	8✔
191	input.Prefix = aws.String(prefix)	3✔
192	}	3✔
193
194	output, err := client.ListObjectsV2(ctx, input)	5✔
195	if err != nil {	5✔
NEW UNCOV 196	return nil, fmt.Errorf("failed to list objects in bucket %s: %w", bucketName, err)	×
NEW UNCOV 197	}	×
198
199	return output.Contents, nil	5✔
200	}
201
202	// DeleteFile deletes an object from S3
203	func (lc *LocalstackTestContainer) DeleteFile(ctx context.Context, bucketName, objectKey string) error {	2✔
204	client, err := lc.createS3Client(ctx)	2✔
205	if err != nil {	2✔
NEW UNCOV 206	return fmt.Errorf("failed to create S3 client: %w", err)	×
NEW UNCOV 207	}	×
208
209	// delete object
210	_, err = client.DeleteObject(ctx, &s3.DeleteObjectInput{	2✔
211	Bucket: aws.String(bucketName),	2✔
212	Key: aws.String(objectKey),	2✔
213	})	2✔
214	if err != nil {	2✔
NEW UNCOV 215	return fmt.Errorf("failed to delete object %s from bucket %s: %w", objectKey, bucketName, err)	×
NEW UNCOV 216	}	×
217
218	return nil	2✔
219	}
220
221	// Close terminates the container
222	func (lc *LocalstackTestContainer) Close(ctx context.Context) error {	5✔
223	return lc.Container.Terminate(ctx)	5✔
224	}	5✔

go-pkgz / testutils / 14371540395

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous