8783924762

Committed 22 Apr 2024 12:21PM UTC coverage: 76.003% (+0.05%) from 75.949%

Build # 8783924762

Build Type

Pull #2139

github

Committed by

kubasobon

Commit Message

use correct type for Azure Role Definitions

Pull Request Pull Request #2139: Add correct user info for select IAM resources

Run Details

61 of 71 new or added lines in 3 files covered. (85.92%)

1 existing line in 1 file now uncovered.

7389 of 9722 relevant lines covered (76.0%)

18.23 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

78.95

/internal/resources/fetching/fetchers/gcp/assets_fetcher.go

// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package fetchers

import (
        "context"
        "fmt"
        "strings"

        "github.com/elastic/elastic-agent-libs/logp"
        "github.com/huandu/xstrings"
        "google.golang.org/protobuf/types/known/structpb"

        "github.com/elastic/cloudbeat/internal/resources/fetching"
        "github.com/elastic/cloudbeat/internal/resources/fetching/cycle"
        "github.com/elastic/cloudbeat/internal/resources/providers/gcplib/inventory"
)

type GcpAssetsFetcher struct {
        log        *logp.Logger
        resourceCh chan fetching.ResourceInfo
        provider   inventory.ServiceAPI
}

type GcpAsset struct {
        Type    string
        SubType string

        ExtendedAsset *inventory.ExtendedGcpAsset `json:"asset,omitempty"`
}

// GcpAssetTypes https://cloud.google.com/asset-inventory/docs/supported-asset-types
// map of types to asset types.
// sub-type is derived from asset type by using the first and last segments of the asset type name
// example: gcp-cloudkms-crypto-key
var GcpAssetTypes = map[string][]string{
        fetching.ProjectManagement: {
                inventory.CrmProjectAssetType,
        },
        fetching.KeyManagement: {
                inventory.ApiKeysKeyAssetType,
                inventory.CloudKmsCryptoKeyAssetType,
        },
        fetching.CloudIdentity: {
                inventory.IamServiceAccountAssetType,
                inventory.IamServiceAccountKeyAssetType,
        },
        fetching.CloudDatabase: {
                inventory.BigqueryDatasetAssetType,
                inventory.BigqueryTableAssetType,
                inventory.SqlDatabaseInstanceAssetType,
        },
        fetching.CloudStorage: {
                inventory.StorageBucketAssetType,
                inventory.LogBucketAssetType,
        },
        fetching.CloudCompute: {
                inventory.ComputeInstanceAssetType,
                inventory.ComputeFirewallAssetType,
                inventory.ComputeDiskAssetType,
                inventory.ComputeNetworkAssetType,
                inventory.ComputeBackendServiceAssetType,
                inventory.ComputeSubnetworkAssetType,
        },
        fetching.CloudDns: {
                inventory.DnsManagedZoneAssetType,
        },
        fetching.DataProcessing: {
                inventory.DataprocClusterAssetType,
        },
}

func NewGcpAssetsFetcher(_ context.Context, log *logp.Logger, ch chan fetching.ResourceInfo, provider inventory.ServiceAPI) *GcpAssetsFetcher {
        return &GcpAssetsFetcher{
                log:        log,
                resourceCh: ch,
                provider:   provider,
        }
}

func (f *GcpAssetsFetcher) Fetch(ctx context.Context, cycleMetadata cycle.Metadata) error {
        f.log.Info("Starting GcpAssetsFetcher.Fetch")

        for typeName, assetTypes := range GcpAssetTypes {
                assets, err := f.provider.ListAllAssetTypesByName(ctx, assetTypes)
                if err != nil {
                        f.log.Errorf("Failed to list assets for type %s: %s", typeName, err.Error())
                        continue
                }

                for _, asset := range assets {
                        select {
                        case <-ctx.Done():
                                f.log.Infof("GcpAssetsFetcher.Fetch context err: %s", ctx.Err().Error())
                                return nil
                        case f.resourceCh <- fetching.ResourceInfo{
                                CycleMetadata: cycleMetadata,
                                Resource: &GcpAsset{
                                        Type:          typeName,
                                        SubType:       getGcpSubType(asset.AssetType),
                                        ExtendedAsset: asset,
                                },
                        }:
                        }
                }
        }

        return nil
}

func (f *GcpAssetsFetcher) Stop() {
        f.provider.Close()
}

func (r *GcpAsset) GetData() any {
        return r.ExtendedAsset.Asset
}

func (r *GcpAsset) GetMetadata() (fetching.ResourceMetadata, error) {
        var region string

        if r.ExtendedAsset.Resource != nil {
                region = r.ExtendedAsset.Resource.Location
        }

        return fetching.ResourceMetadata{
                ID:                   r.ExtendedAsset.Name,
                Type:                 r.Type,
                SubType:              r.SubType,
                Name:                 getAssetResourceName(r.ExtendedAsset),
                Region:               region,
                CloudAccountMetadata: *r.ExtendedAsset.CloudAccount,
        }, nil
}

func (r *GcpAsset) GetElasticCommonData() (map[string]any, error) {
        m := map[string]any{}

        if r.Type == fetching.CloudIdentity {
                m["user.effective.id"] = r.ExtendedAsset.Name
                m["user.effective.name"] = getAssetResourceName(r.ExtendedAsset)
        }

        if r.Type == fetching.CloudCompute && r.SubType == inventory.ComputeInstanceAssetType {
                fields := getAssetDataFields(r.ExtendedAsset)
                if fields == nil {
                        return m, nil
                }
                nameField, ok := fields["name"]
                if ok {
                        if name := nameField.GetStringValue(); name != "" {
                                m["host.name"] = name
                        }
                }
                hostnameField, ok := fields["hostname"]
                if ok {
                        if hostname := hostnameField.GetStringValue(); hostname != "" {
                                m["host.hostname"] = hostname
                        }
                }
        }

        return m, nil
}

// try to retrieve the resource name from the asset data fields (name or displayName), in case it is not set
// get the last part of the asset name (https://cloud.google.com/apis/design/resource_names#resource_id)
func getAssetResourceName(asset *inventory.ExtendedGcpAsset) string {
        fields := getAssetDataFields(asset)
        if fields != nil {
                if name, exist := fields["displayName"]; exist && name.GetStringValue() != "" {
                        return name.GetStringValue()
                }

                if name, exist := fields["name"]; exist && name.GetStringValue() != "" {
                        return name.GetStringValue()
                }
        }
        parts := strings.Split(asset.Name, "/")
        return parts[len(parts)-1]
}

func getGcpSubType(assetType string) string {
        dotIndex := strings.Index(assetType, ".")
        slashIndex := strings.Index(assetType, "/")

        prefix := assetType[:dotIndex]
        suffix := assetType[slashIndex+1:]

        return strings.ToLower(fmt.Sprintf("gcp-%s-%s", prefix, xstrings.ToKebabCase(suffix)))
}

// getAssetDataFields tries to retrieve asset.resource.data fields if possible.
// Returns nil otherwise.
func getAssetDataFields(asset *inventory.ExtendedGcpAsset) map[string]*structpb.Value {
        resource := asset.GetResource()
        if resource == nil {
                return nil
        }
        data := resource.GetData()
        if data == nil {
                return nil
        }
        return data.GetFields()
}

1	// Licensed to Elasticsearch B.V. under one or more contributor
2	// license agreements. See the NOTICE file distributed with
3	// this work for additional information regarding copyright
4	// ownership. Elasticsearch B.V. licenses this file to you under
5	// the Apache License, Version 2.0 (the "License"); you may
6	// not use this file except in compliance with the License.
7	// You may obtain a copy of the License at
8	//
9	// http://www.apache.org/licenses/LICENSE-2.0
10	//
11	// Unless required by applicable law or agreed to in writing,
12	// software distributed under the License is distributed on an
13	// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14	// KIND, either express or implied. See the License for the
15	// specific language governing permissions and limitations
16	// under the License.
17
18	package fetchers
19
20	import (
21	"context"
22	"fmt"
23	"strings"
24
25	"github.com/elastic/elastic-agent-libs/logp"
26	"github.com/huandu/xstrings"
27	"google.golang.org/protobuf/types/known/structpb"
28
29	"github.com/elastic/cloudbeat/internal/resources/fetching"
30	"github.com/elastic/cloudbeat/internal/resources/fetching/cycle"
31	"github.com/elastic/cloudbeat/internal/resources/providers/gcplib/inventory"
32	)
33
34	type GcpAssetsFetcher struct {
35	log *logp.Logger
36	resourceCh chan fetching.ResourceInfo
37	provider inventory.ServiceAPI
38	}
39
40	type GcpAsset struct {
41	Type string
42	SubType string
43
44	ExtendedAsset *inventory.ExtendedGcpAsset `json:"asset,omitempty"`
45	}
46
47	// GcpAssetTypes https://cloud.google.com/asset-inventory/docs/supported-asset-types
48	// map of types to asset types.
49	// sub-type is derived from asset type by using the first and last segments of the asset type name
50	// example: gcp-cloudkms-crypto-key
51	var GcpAssetTypes = map[string][]string{
52	fetching.ProjectManagement: {
53	inventory.CrmProjectAssetType,
54	},
55	fetching.KeyManagement: {
56	inventory.ApiKeysKeyAssetType,
57	inventory.CloudKmsCryptoKeyAssetType,
58	},
59	fetching.CloudIdentity: {
60	inventory.IamServiceAccountAssetType,
61	inventory.IamServiceAccountKeyAssetType,
62	},
63	fetching.CloudDatabase: {
64	inventory.BigqueryDatasetAssetType,
65	inventory.BigqueryTableAssetType,
66	inventory.SqlDatabaseInstanceAssetType,
67	},
68	fetching.CloudStorage: {
69	inventory.StorageBucketAssetType,
70	inventory.LogBucketAssetType,
71	},
72	fetching.CloudCompute: {
73	inventory.ComputeInstanceAssetType,
74	inventory.ComputeFirewallAssetType,
75	inventory.ComputeDiskAssetType,
76	inventory.ComputeNetworkAssetType,
77	inventory.ComputeBackendServiceAssetType,
78	inventory.ComputeSubnetworkAssetType,
79	},
80	fetching.CloudDns: {
81	inventory.DnsManagedZoneAssetType,
82	},
83	fetching.DataProcessing: {
84	inventory.DataprocClusterAssetType,
85	},
86	}
87
88	func NewGcpAssetsFetcher(_ context.Context, log logp.Logger, ch chan fetching.ResourceInfo, provider inventory.ServiceAPI) GcpAssetsFetcher {	2✔
89	return &GcpAssetsFetcher{	2✔
90	log: log,	2✔
91	resourceCh: ch,	2✔
92	provider: provider,	2✔
93	}	2✔
94	}	2✔
95
96	func (f *GcpAssetsFetcher) Fetch(ctx context.Context, cycleMetadata cycle.Metadata) error {	1✔
97	f.log.Info("Starting GcpAssetsFetcher.Fetch")	1✔
98		1✔
99	for typeName, assetTypes := range GcpAssetTypes {	9✔
100	assets, err := f.provider.ListAllAssetTypesByName(ctx, assetTypes)	8✔
101	if err != nil {	8✔
102	f.log.Errorf("Failed to list assets for type %s: %s", typeName, err.Error())	×
103	continue	×
104	}
105
106	for _, asset := range assets {	16✔
107	select {	8✔
108	case <-ctx.Done():	×
109	f.log.Infof("GcpAssetsFetcher.Fetch context err: %s", ctx.Err().Error())	×
110	return nil	×
111	case f.resourceCh <- fetching.ResourceInfo{
112	CycleMetadata: cycleMetadata,
113	Resource: &GcpAsset{
114	Type: typeName,
115	SubType: getGcpSubType(asset.AssetType),
116	ExtendedAsset: asset,
117	},
118	}:	8✔
119	}
120	}
121	}
122
123	return nil	1✔
124	}
125
126	func (f *GcpAssetsFetcher) Stop() {	2✔
127	f.provider.Close()	2✔
128	}	2✔
129
130	func (r *GcpAsset) GetData() any {	×
131	return r.ExtendedAsset.Asset	×
132	}	×
133
134	func (r *GcpAsset) GetMetadata() (fetching.ResourceMetadata, error) {	8✔
135	var region string	8✔
136		8✔
137	if r.ExtendedAsset.Resource != nil {	8✔
138	region = r.ExtendedAsset.Resource.Location	×
139	}	×
140
141	return fetching.ResourceMetadata{	8✔
142	ID: r.ExtendedAsset.Name,	8✔
143	Type: r.Type,	8✔
144	SubType: r.SubType,	8✔
145	Name: getAssetResourceName(r.ExtendedAsset),	8✔
146	Region: region,	8✔
147	CloudAccountMetadata: *r.ExtendedAsset.CloudAccount,	8✔
148	}, nil	8✔
149	}
150
151	func (r *GcpAsset) GetElasticCommonData() (map[string]any, error) {	7✔
152	m := map[string]any{}	7✔
153		7✔
154	if r.Type == fetching.CloudIdentity {	8✔
155	m["user.effective.id"] = r.ExtendedAsset.Name	1✔
156	m["user.effective.name"] = getAssetResourceName(r.ExtendedAsset)	1✔
157	}	1✔
158
159	if r.Type == fetching.CloudCompute && r.SubType == inventory.ComputeInstanceAssetType {	13✔
160	fields := getAssetDataFields(r.ExtendedAsset)	6✔
161	if fields == nil {	6✔
NEW 162	return m, nil	×
UNCOV 163	}	×
164	nameField, ok := fields["name"]	6✔
165	if ok {	9✔
166	if name := nameField.GetStringValue(); name != "" {	5✔
167	m["host.name"] = name	2✔
168	}	2✔
169	}
170	hostnameField, ok := fields["hostname"]	6✔
171	if ok {	9✔
172	if hostname := hostnameField.GetStringValue(); hostname != "" {	5✔
173	m["host.hostname"] = hostname	2✔
174	}	2✔
175	}
176	}
177
178	return m, nil	7✔
179	}
180
181	// try to retrieve the resource name from the asset data fields (name or displayName), in case it is not set
182	// get the last part of the asset name (https://cloud.google.com/apis/design/resource_names#resource_id)
183	func getAssetResourceName(asset *inventory.ExtendedGcpAsset) string {	9✔
184	fields := getAssetDataFields(asset)	9✔
185	if fields != nil {	9✔
NEW 186	if name, exist := fields["displayName"]; exist && name.GetStringValue() != "" {	×
NEW 187	return name.GetStringValue()	×
NEW 188	}	×
189
NEW 190	if name, exist := fields["name"]; exist && name.GetStringValue() != "" {	×
NEW 191	return name.GetStringValue()	×
NEW 192	}	×
193	}
194	parts := strings.Split(asset.Name, "/")	9✔
195	return parts[len(parts)-1]	9✔
196	}
197
198	func getGcpSubType(assetType string) string {	8✔
199	dotIndex := strings.Index(assetType, ".")	8✔
200	slashIndex := strings.Index(assetType, "/")	8✔
201		8✔
202	prefix := assetType[:dotIndex]	8✔
203	suffix := assetType[slashIndex+1:]	8✔
204		8✔
205	return strings.ToLower(fmt.Sprintf("gcp-%s-%s", prefix, xstrings.ToKebabCase(suffix)))	8✔
206	}	8✔
207
208	// getAssetDataFields tries to retrieve asset.resource.data fields if possible.
209	// Returns nil otherwise.
210	func getAssetDataFields(asset inventory.ExtendedGcpAsset) map[string]structpb.Value {	15✔
211	resource := asset.GetResource()	15✔
212	if resource == nil {	24✔
213	return nil	9✔
214	}	9✔
215	data := resource.GetData()	6✔
216	if data == nil {	6✔
NEW 217	return nil	×
NEW 218	}	×
219	return data.GetFields()	6✔
220	}

elastic / cloudbeat / 8783924762

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous