13989782125

Committed 21 Mar 2025 10:32AM UTC coverage: 75.671% (+0.05%) from 75.619%

Build # 13989782125

Build Type

push

github

Committed by

web-flow

Commit Message

[8.x](backport #3090) cnvm: Delete snapshots after scanning them (#3127)

### Summary of your changes
Fixes various underlying issues with CNVM snapshot deletion. The logic here is to do a best-effort attempt to clean up snapshots created during the run both continuously (after we are done scanning the snapshot) and on shutdown. Cleaning old snapshots that we don't use anymore is part of https://github.com/elastic/cloudbeat/issues/3105. Issues fixed:
- `internal/flavors/vulnerability.go`: Wait for `Run()` to finish, this ensures that final snapshot clean-up is done after execution finishes
- `internal/resources/providers/awslib/ec2/provider.go`: Give extra retries to snapshot deletion, mainly avoiding "too many requests" errors
- `internal/vulnerability/snapshot.go`: New snapshot manager to handle creation, deletion and clean-up of snapshots. The deletion extends the `context.Context` with an extra 30s timeout to give a grace period to clean-up snapshots during shutdown/restart.
- `internal/vulnerability/replicator.go`: Add dependency to the snapshot manager instead of `provider` to track created snapshots
- `internal/vulnerability/scanner.go`: Delete snapshot after scanning
- `internal/vulnerability/worker.go`: `defer` a call snapshot manager's cleanup

### Screenshot/Data
1. The way I verified we avoid leftover snapshots is to change the name of the snapshots:
  ```diff
  diff --git a/internal/resources/providers/awslib/ec2/provider.go b/internal/resources/providers/awslib/ec2/provider.go
  index 14abc5bf..3faeef7d 100644
  --- a/internal/resources/providers/awslib/ec2/provider.go
  +++ b/internal/resources/providers/awslib/ec2/provider.go
  @@ -78,7 +78,7 @@ func (p *Provider) CreateSnapshots(ctx context.Context, ins *Ec2Instance) ([]EBS
 			  {
 				  ResourceType: "snapshot",
 				  Tags: []types.Tag{
  -					{Key: aws.String("Name"), Value: aws.String(fmt.Sprintf("elastic-vulnerability-%s", *ins.InstanceId))},
  +					{Key: aws.String("Name"), V... (continued)

Run Details

151 of 171 new or added lines in 10 files covered. (88.3%)

1 existing line in 1 file now uncovered.

8989 of 11879 relevant lines covered (75.67%)

16.21 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

86.11

/internal/vulnerability/replicator.go

// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package vulnerability

import (
        "context"

        "github.com/elastic/cloudbeat/internal/infra/clog"
        "github.com/elastic/cloudbeat/internal/resources/providers/awslib/ec2"
)

type VulnerabilityReplicator struct {
        log     *clog.Logger
        manager *SnapshotManager
        ch      chan ec2.EBSSnapshot
}

func NewVulnerabilityReplicator(log *clog.Logger, manager *SnapshotManager) VulnerabilityReplicator {
        log.Debug("VulnerabilityReplicator: New")
        ch := make(chan ec2.EBSSnapshot)
        return VulnerabilityReplicator{
                log:     log,
                ch:      ch,
                manager: manager,
        }
}

func (f VulnerabilityReplicator) SnapshotInstance(ctx context.Context, insCh chan *ec2.Ec2Instance) {
        f.log.Info("Starting VulnerabilityReplicator.SnapshotInstance")
        defer close(f.ch)
        for {
                select {
                case <-ctx.Done():
                        f.log.Info("VulnerabilityReplicator.SnapshotInstance context canceled")
                        return
                case data, ok := <-insCh:
                        if !ok {
                                f.log.Info("VulnerabilityReplicator.SnapshotInstance channel is closed")
                                return
                        }
                        sp, err := f.manager.CreateSnapshots(ctx, data)
                        if err != nil {
                                f.log.Errorf("VulnerabilityReplicator.SnapshotInstance.CreateSnapshots failed: %v", err)
                                continue
                        }

                        for _, s := range sp {
                                f.log.Infof("VulnerabilityReplicator.SnapshotInstance created snapshot: %s of size %d for instance %s", s.SnapshotId, s.VolumeSize, *data.InstanceId)
                                select {
                                case <-ctx.Done():
                                        f.log.Info("VulnerabilityReplicator.SnapshotInstance context canceled")
                                        return
                                case f.ch <- s:
                                }
                        }
                }
        }
}

func (f VulnerabilityReplicator) GetChan() chan ec2.EBSSnapshot {
        return f.ch
}

1	// Licensed to Elasticsearch B.V. under one or more contributor
2	// license agreements. See the NOTICE file distributed with
3	// this work for additional information regarding copyright
4	// ownership. Elasticsearch B.V. licenses this file to you under
5	// the Apache License, Version 2.0 (the "License"); you may
6	// not use this file except in compliance with the License.
7	// You may obtain a copy of the License at
8	//
9	// http://www.apache.org/licenses/LICENSE-2.0
10	//
11	// Unless required by applicable law or agreed to in writing,
12	// software distributed under the License is distributed on an
13	// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14	// KIND, either express or implied. See the License for the
15	// specific language governing permissions and limitations
16	// under the License.
17
18	package vulnerability
19
20	import (
21	"context"
22
23	"github.com/elastic/cloudbeat/internal/infra/clog"
24	"github.com/elastic/cloudbeat/internal/resources/providers/awslib/ec2"
25	)
26
27	type VulnerabilityReplicator struct {
28	log *clog.Logger
29	manager *SnapshotManager
30	ch chan ec2.EBSSnapshot
31	}
32
33	func NewVulnerabilityReplicator(log clog.Logger, manager SnapshotManager) VulnerabilityReplicator {	4✔
34	log.Debug("VulnerabilityReplicator: New")	4✔
35	ch := make(chan ec2.EBSSnapshot)	4✔
36	return VulnerabilityReplicator{	4✔
37	log: log,	4✔
38	ch: ch,	4✔
39	manager: manager,	4✔
40	}	4✔
41	}	4✔
42
43	func (f VulnerabilityReplicator) SnapshotInstance(ctx context.Context, insCh chan *ec2.Ec2Instance) {	3✔
44	f.log.Info("Starting VulnerabilityReplicator.SnapshotInstance")	3✔
45	defer close(f.ch)	3✔
46	for {	8✔
47	select {	5✔
48	case <-ctx.Done():	×
49	f.log.Info("VulnerabilityReplicator.SnapshotInstance context canceled")	×
NEW 50	return	×
51	case data, ok := <-insCh:	5✔
52	if !ok {	7✔
53	f.log.Info("VulnerabilityReplicator.SnapshotInstance channel is closed")	2✔
54	return	2✔
55	}	2✔
56	sp, err := f.manager.CreateSnapshots(ctx, data)	3✔
57	if err != nil {	3✔
58	f.log.Errorf("VulnerabilityReplicator.SnapshotInstance.CreateSnapshots failed: %v", err)	×
59	continue	×
60	}
61
62	for _, s := range sp {	6✔
63	f.log.Infof("VulnerabilityReplicator.SnapshotInstance created snapshot: %s of size %d for instance %s", s.SnapshotId, s.VolumeSize, *data.InstanceId)	3✔
64	select {	3✔
65	case <-ctx.Done():	1✔
66	f.log.Info("VulnerabilityReplicator.SnapshotInstance context canceled")	1✔
67	return	1✔
68	case f.ch <- s:	2✔
69	}
70	}
71	}
72	}
73	}
74
75	func (f VulnerabilityReplicator) GetChan() chan ec2.EBSSnapshot {	3✔
76	return f.ch	3✔
77	}	3✔

elastic / cloudbeat / 13989782125

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous